Added logic to fetch README files, documentation, commit messages, an…

website/api/views.py

@@ -711,7 +711,7 @@ class ProjectViewSet(viewsets.ModelViewSet):
     queryset = Project.objects.all()
     serializer_class = ProjectSerializer
     # permission_classes = (IsAuthenticatedOrReadOnly,)
-    http_method_names = ("get", "post")
+    http_method_names = ("get", "post", "put", "patch")
 
     def create(self, request, *args, **kwargs):
         data = request.data.copy()
@@ -737,6 +737,20 @@ def create(self, request, *args, **kwargs):
 
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
+    def update(self, request, *args, **kwargs):
+        partial = kwargs.pop("partial", False)
+        instance = self.get_object()
+
+        # Allow maintainers to manually adjust `ai_summary` and `ai_labels`
+        data = request.data.copy()
+
+        # Update the instance with new data, including `ai_summary` and `ai_labels`
+        serializer = ProjectSerializer(instance, data=data, partial=partial)
+        serializer.is_valid(raise_exception=True)
+        serializer.save()
+
+        return Response(serializer.data, status=status.HTTP_200_OK)
+
     def list(self, request, *args, **kwargs):
         projects = Project.objects.prefetch_related("contributors").all()
 

website/label_generation.py

@@ -0,0 +1,43 @@
+import os
+
+import openai
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+
+def generate_labels(readme_content, github_topics):
+    prompt = f"""
+    You are an AI that assigns relevant labels to GitHub projects based on their readme content and github topics.
+
+    ### Input:
+    - **README Content:** {readme_content}
+    - **GitHub Topics:** {github_topics}
+
+    ### Task:
+    Analyze the input and assign appropriate labels. 
+    Labels should include:
+    1. **Technology Stacks** (e.g., Python, JavaScript, Java).
+    2. **Project Type** (e.g., Web Application, CLI Tool, Library).
+    3. **OWASP Relevance** (e.g., Security Testing, Secure Coding).
+    4. Any other relevant labels.
+
+    ### Output:
+    Provide the labels in JSON format like this:
+    {{
+      "tech-stack": ["Python", "JavaScript"],
+      "project-type": ["Web Application"],
+      "owasp-relevance": ["Secure Coding"],
+      "other": ["Machine Learning", "Data Processing"]
+    }}
+    """
+
+    response = openai.ChatCompletion.create(
+        model="gpt-4",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant for labeling projects."},
+            {"role": "user", "content": prompt},
+        ],
+        temperature=0.7,
+    )
+
+    return response["choices"][0]["message"]["content"]

website/management/commands/update_projects.py

@@ -1,9 +1,15 @@
+import base64
+import json
+
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 
+from website.label_generation import generate_labels
 from website.models import Project
+from website.summarization import ai_summary
+from website.utils import markdown_to_text
 
 
 class Command(BaseCommand):
@@ -50,6 +56,52 @@ def handle(self, *args, **kwargs):
                 project.updated_at = parse_datetime(repo_data.get("updated_at"))
                 project.size = repo_data.get("size", 0)
                 project.last_commit_date = parse_datetime(repo_data.get("pushed_at"))
+                project.topics = repo_data.get("topics", [])
+
+                # Fetch README
+                url = f"https://api.github.com/repos/{repo_name}/readme"
+                response = requests.get(url, headers=headers)
+                if response.status_code == 200:
+                    readme_data = response.json()
+                    readme_content_encoded = readme_data.get("content", "")
+
+                    # Decode the Base64 content
+                    try:
+                        readme_content = base64.b64decode(readme_content_encoded).decode("utf-8")
+                        project.readme_content = readme_content
+                        readme_text = markdown_to_text(readme_content)
+                        project.ai_summary = ai_summary(readme_text, project.topics)
+                        project.ai_labels = json.loads(generate_labels(readme_text, project.topics))
+                    except (base64.binascii.Error, UnicodeDecodeError) as e:
+                        self.stdout.write(
+                            self.style.WARNING(f"Failed to decode README for {repo_name}: {e}")
+                        )
+                        project.readme_content = ""
+                        project.ai_summary = ""
+                        project.ai_labels = {}
+                else:
+                    self.stdout.write(
+                        self.style.WARNING(
+                            f"Failed to fetch README for {repo_name}: {response.status_code}"
+                        )
+                    )
+
+                # Check for Documentation URL (homepage)
+                project.documentation_url = repo_data.get("homepage")
+
+                # Fetch Recent Commit Messages
+                url = f"https://api.github.com/repos/{repo_name}/commits"
+                response = requests.get(url, headers=headers)
+                if response.status_code == 200:
+                    commits_data = response.json()
+                    commit_messages = [commit["commit"]["message"] for commit in commits_data[:5]]
+                    project.recent_commit_messages = "\n".join(commit_messages)
+                else:
+                    self.stdout.write(
+                        self.style.WARNING(
+                            f"Failed to fetch recent commits for {repo_name}: {response.status_code}"
+                        )
+                    )
 
                 # Fetch counts of issues and pull requests using the Search API
                 def get_issue_count(repo_name, query, headers):

website/migrations/0154_project_ai_labels_project_ai_summary_and_more.py

@@ -0,0 +1,42 @@
+# Generated by Django 5.1.3 on 2024-11-19 08:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0153_delete_contributorstats"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="project",
+            name="ai_labels",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="ai_summary",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="documentation_url",
+            field=models.URLField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="issue_tracker_url",
+            field=models.URLField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="readme_content",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="recent_commit_messages",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

website/migrations/0155_project_topics_alter_project_ai_labels.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.3 on 2024-11-20 09:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0154_project_ai_labels_project_ai_summary_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="project",
+            name="topics",
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="project",
+            name="ai_labels",
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

website/models.py

@@ -754,6 +754,13 @@ class Project(models.Model):
     closed_issues = models.IntegerField(default=0)
     size = models.IntegerField(default=0)
     commit_count = models.IntegerField(default=0)
+    readme_content = models.TextField(null=True, blank=True)
+    documentation_url = models.URLField(null=True, blank=True)
+    recent_commit_messages = models.TextField(null=True, blank=True)
+    issue_tracker_url = models.URLField(null=True, blank=True)
+    topics = models.JSONField(null=True, blank=True)
+    ai_summary = models.TextField(null=True, blank=True)
+    ai_labels = models.JSONField(null=True, blank=True)
 
     def __str__(self):
         return self.name

website/summarization.py

@@ -0,0 +1,22 @@
+import os
+
+import openai
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+
+def ai_summary(text, topics=None):
+    """Generate an AI-driven summary using OpenAI's GPT, including GitHub topics."""
+    try:
+        topics_str = ", ".join(topics) if topics else "No topics provided."
+        prompt = f"Generate a brief summary of the following text, focusing on key aspects such as purpose, features, technologies used, and current status. Consider the following GitHub topics to enhance the context: {topics_str}\n\n{text}"
+        response = openai.Completion.create(
+            model="text-davinci-003",
+            prompt=prompt,
+            max_tokens=150,
+            temperature=0.5,
+        )
+        summary = response.choices[0].text.strip()
+        return summary
+    except Exception as e:
+        return f"Error generating summary: {str(e)}"

website/templates/includes/project_list_items.html

@@ -0,0 +1,131 @@
+{% load humanize %}
+<ul class="project-list">
+    {% for project in projects %}
+        <a href="{% url 'project_view' project.slug %}" class="project-link">
+            <li class="project-item">
+                {% if project.logo_url %}
+                    <img src="{{ project.logo_url }}"
+                         alt="{{ project.name }} logo"
+                         class="project-logo"
+                         height="100"
+                         width="100">
+                {% else %}
+                    <div class="project-logo placeholder">
+                        <span>No Logo</span>
+                    </div>
+                {% endif %}
+                <div class="project-details">
+                    <h3>{{ project.name }}</h3>
+                    <!-- Stats Icons Under Project Title -->
+                    <div class="stats-icons">
+                        <ul>
+                            <li>
+                                <a href="{{ project.github_url }}" target="_blank" title="GitHub">
+                                    <i class="fab fa-github"></i>
+                                </a>
+                            </li>
+                            {% if project.wiki_url %}
+                                <li>
+                                    <a href="{{ project.wiki_url }}" target="_blank" title="Wiki">
+                                        <i class="fas fa-book"></i>
+                                    </a>
+                                </li>
+                            {% endif %}
+                            {% if project.homepage_url %}
+                                <li>
+                                    <a href="{{ project.homepage_url }}" target="_blank" title="Homepage">
+                                        <i class="fas fa-home"></i>
+                                    </a>
+                                </li>
+                            {% endif %}
+                            <li>
+                                <a href="{{ project.github_url }}/stargazers"
+                                   target="_blank"
+                                   title="Stars">
+                                    <i class="fas fa-star"></i> {{ project.stars|intcomma }}
+                                </a>
+                            </li>
+                            <li>
+                                <a href="{{ project.github_url }}/network/members"
+                                   target="_blank"
+                                   title="Forks">
+                                    <i class="fas fa-code-branch"></i> {{ project.forks|intcomma }}
+                                </a>
+                            </li>
+                            <li>
+                                <a href="{{ project.github_url }}/issues"
+                                   target="_blank"
+                                   title="Total Issues">
+                                    <i class="fas fa-exclamation-circle"></i> {{ project.total_issues|intcomma }}
+                                </a>
+                            </li>
+                            {% if project.release_name %}
+                                <li>
+                                    <i class="fas fa-tag"></i>
+                                    <span title="{{ project.release_datetime|date:'Y-m-d H:i' }}">{{ project.release_name }}</span>
+                                </li>
+                            {% endif %}
+                            <li>
+                                <i class="fas fa-clock"></i>
+                                <span title="{{ project.last_updated|date:'Y-m-d H:i' }}">{{ project.last_updated|timesince }}</span>
+                            </li>
+                        </ul>
+                    </div>
+                    <p>{{ project.description }}</p>
+                    <p class="black-text">{{ project.ai_summary }}</p>
+                    <div class="label-group">
+                        {% for category, labels in project.ai_labels.items %}
+                            {% for label in labels %}<span class="label {{ category }}">{{ label }}</span>{% endfor %}
+                        {% endfor %}
+                    </div>
+                    <!-- Contributors Section -->
+                    <div class="project-stats">
+                        <p>
+                            <i class="fas fa-users"></i> Contributors: {{ project.contributor_count }}
+                        </p>
+                        <div class="contributors">
+                            {% for contributor in project.get_top_contributors %}
+                                <a href="https://github.com/search?q=repo:{{ project.github_url|slice:'19:' }}+author:{{ contributor.name }}&type=issues"
+                                   target="_blank">
+                                    <img src="{{ contributor.avatar_url }}"
+                                         alt="{{ contributor.name }}"
+                                         class="contributor-avatar"
+                                         title="{{ contributor.name }}"
+                                         height="40"
+                                         width="40">
+                                </a>
+                            {% empty %}
+                                <div class="contributor-avatar placeholder">
+                                    <span>No Avatar</span>
+                                </div>
+                            {% endfor %}
+                        </div>
+                    </div>
+                    <div class="additional-metadata">
+                        <div class="external-links">
+                            <h4>
+                                <i class="fas fa-external-link-alt"></i> Links
+                            </h4>
+                            <ul>
+                                {% for link in project.external_links %}
+                                    <li>
+                                        <a href="{{ link.url }}" target="_blank"><i class="fas fa-link"></i> {{ link.name }}</a>
+                                    </li>
+                                {% endfor %}
+                            </ul>
+                        </div>
+                    </div>
+                    <!-- Tags Section -->
+                    <div class="project-tags">
+                        <h4>
+                            <i class="fas fa-tags"></i>Tags
+                        </h4>
+                        <ul>
+                            {% for tag in project.tags.all %}<li>{{ tag.name }}</li>{% endfor %}
+                        </ul>
+                    </div>
+                </div>
+            </li>
+        </a>
+    {% endfor %}
+</ul>