Add datasource ID

PIPELINES-AVID.rst

@@ -1,11 +1,13 @@
-.. list-table:: Pipeline AVID Mapping
+.. list-table:: Pipeline Advisory UID Mapping
    :header-rows: 1
    :widths: 35 65
 
    * - pipeline name
-     - AVID
+     - Advisory UID
+     - datasource name
    * - alpine_linux_importer_v2
      - {package_name}/{distroversion}/{version}/{vulnerability_id}
+     - alpine_linux
    * - aosp_dataset_fix_commits
      - CVE ID of the record
    * - apache_httpd_importer_v2

vulnerabilities/migrations/0130_advisoryv2_pipeline_id.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.2.11 on 2026-05-18 08:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0129_advisorypoc"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=200,
+                null=True,
+            ),
+        ),
+    ]

vulnerabilities/migrations/0131_auto_20260518_0854.py

@@ -0,0 +1,42 @@
+# Generated by Django 5.2.11 on 2026-05-18 08:54
+
+from django.db import migrations, models
+from django.db.models import F
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0130_advisoryv2_pipeline_id"),
+    ]
+
+    def populate_pipeline_id(apps, schema_editor):
+        Advisory = apps.get_model("vulnerabilities", "AdvisoryV2")
+
+        Advisory.objects.update(
+            pipeline_id=F("datasource_id")
+        )
+
+        assert not Advisory.objects.filter(pipeline_id="").exists(), "Some advisories have an empty pipeline_id after the update"
+
+    operations = [
+        migrations.RunPython(populate_pipeline_id, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=200,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="datasource_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the datasource used for this advisory .e.g.: nginx",
+                max_length=200,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -3058,7 +3058,15 @@ class AdvisoryV2(models.Model):
         blank=False,
         null=False,
         db_index=True,
-        help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx_importer_v2",
+        help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx",
+    )
+
+    pipeline_id = models.CharField(
+        max_length=200,
+        blank=False,
+        null=False,
+        db_index=True,
+        help_text="Unique ID for the pipeline used for this advisory ." "e.g.: nginx_importer_v2",
     )
 
     # This is similar to a name

vulnerabilities/pipelines/__init__.py

@@ -273,7 +273,7 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
 
     pipeline_id = None  # Unique Pipeline ID, this should be the name of pipeline module.
     license_url = None
-    datasource_name = None
+    datasource_id = None
     spdx_license_expression = None
     repo_url = None
     ignorable_versions = []
@@ -319,6 +319,9 @@ def advisories_count(self) -> int:
         raise NotImplementedError
 
     def collect_and_store_advisories(self):
+        if not self.pipeline_id and not self.datasource_id:
+            self.log("Pipeline must have a unique pipeline_id or datasource_id defined.")
+            return
         collected_advisory_count = 0
         estimated_advisory_count = self.advisories_count()
 
@@ -338,6 +341,7 @@ def collect_and_store_advisories(self):
                 if _obj := insert_advisory_v2(
                     advisory=advisory,
                     pipeline_id=self.pipeline_id,
+                    datasource_id=self.datasource_id,
                     logger=self.log,
                     precedence=self.precedence,
                 ):

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -38,6 +38,7 @@ class AlpineLinuxImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
 
     pipeline_id = "alpine_linux_importer_v2"
     spdx_license_expression = "CC-BY-SA-4.0"
+    datasource_id = "alpine_linux"
     license_url = "https://secdb.alpinelinux.org/license.txt"
     repo_url = "git+https://github.com/aboutcode-org/aboutcode-mirror-alpine-secdb/"
 

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -28,6 +28,7 @@ class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "aosp_dataset_fix_commits"
+    datasource_id = "aosp_dataset"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE"
 

vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py

@@ -148,6 +148,7 @@ class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "apache_httpd_importer_v2"
+    datasource_id = "apache_httpd"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://www.apache.org/licenses/LICENSE-2.0"
     base_url = "https://httpd.apache.org/security/json/"

vulnerabilities/pipelines/v2_importers/apache_kafka_importer.py

@@ -33,6 +33,7 @@ class ApacheKafkaImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """Import Apache Kafka Advisories"""
 
     pipeline_id = "apache_kafka_importer_v2"
+    datasource_id = "apache_kafka"
     spdx_license_expression = "Apache-2.0"
     importer_name = "Apache Kafka Importer V2"
 

vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py

@@ -36,6 +36,7 @@ class ApacheTomcatImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "apache_tomcat_importer_v2"
+    datasource_id = "apache_tomcat"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://www.apache.org/licenses/LICENSE-2.0"
     base_url = "https://tomcat.apache.org/security"

vulnerabilities/pipelines/v2_importers/archlinux_importer.py

@@ -29,6 +29,7 @@ class ArchLinuxImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
 
     pipeline_id = "archlinux_importer_v2"
     spdx_license_expression = "MIT"
+    datasource_id = "archlinux"
     license_url = "https://github.com/archlinux/arch-security-tracker/blob/master/LICENSE"
 
     precedence = 200

vulnerabilities/pipelines/v2_importers/collect_fix_commits.py

@@ -3,184 +3,221 @@
 
 class CollectLinuxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_linux_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/torvalds/linux"
 
 
 class CollectBusyBoxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_busybox_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/mirror/busybox"
 
 
 class CollectNginxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_nginx_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/nginx/nginx"
 
 
 class CollectApacheTomcatFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_apache_tomcat_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/apache/tomcat"
 
 
 class CollectMysqlServerFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_mysql_server_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/mysql/mysql-server"
 
 
 class CollectPostgresqlFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_postgresql_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/postgres/postgres"
 
 
 class CollectMongodbFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_mongodb_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/mongodb/mongo"
 
 
 class CollectRedisFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_redis_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/redis/redis"
 
 
 class CollectSqliteFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_sqlite_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/sqlite/sqlite"
 
 
 class CollectPhpFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_php_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/php/php-src"
 
 
 class CollectPythonCpythonFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_python_cpython_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/python/cpython"
 
 
 class CollectRubyFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_ruby_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/ruby/ruby"
 
 
 class CollectGoFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_go_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/golang/go"
 
 
 class CollectNodeJsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_node_js_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/nodejs/node"
 
 
 class CollectRustFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_rust_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/rust-lang/rust"
 
 
 class CollectOpenjdkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_openjdk_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/openjdk/jdk"
 
 
 class CollectSwiftFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_swift_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/swiftlang/swift"
 
 
 class CollectDjangoFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_django_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/django/django"
 
 
 class CollectRailsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_rails_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/rails/rails"
 
 
 class CollectLaravelFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_laravel_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/laravel/framework"
 
 
 class CollectSpringFrameworkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_spring_framework_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/spring-projects/spring-framework"
 
 
 class CollectReactFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_react_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/facebook/react"
 
 
 class CollectAngularFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_angular_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/angular/angular"
 
 
 class CollectWordpressFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_wordpress_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/WordPress/WordPress"
 
 
 class CollectDockerMobyFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_docker_moby_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/moby/moby"
 
 
 class CollectKubernetesFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_kubernetes_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/kubernetes/kubernetes"
 
 
 class CollectQemuFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_qemu_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://gitlab.com/qemu-project/qemu"
 
 
 class CollectXenProjectFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_xen_project_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/xen-project/xen"
 
 
 class CollectVirtualboxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_virtualbox_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/mirror/vbox"
 
 
 class CollectContainerdFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_containerd_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/containerd/containerd"
 
 
 class CollectAnsibleFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_ansible_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/ansible/ansible"
 
 
 class CollectTerraformFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_terraform_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/hashicorp/terraform"
 
 
 class CollectWiresharkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_wireshark_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://gitlab.com/wireshark/wireshark"
 
 
 class CollectTcpdumpFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_tcpdump_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/the-tcpdump-group/tcpdump"
 
 
 class CollectGitFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_git_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/git/git"
 
 
 class CollectJenkinsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_jenkins_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://github.com/jenkinsci/jenkins"
 
 
 class CollectGitlabFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_gitlab_fix_commits"
+    datasource_id = pipeline_id.replace("collect_", "")
     repo_url = "https://gitlab.com/gitlab-org/gitlab-foss"