Federation (#133)

* mtu, vlan id and prefixes

* make prefix unique

* prefix form cleanup

* linting

* expose prefix to service bridge

* expose ix field

* netbox sync

* linting

* docs

* docs

* fix tests

* Update to federation

* relock

---------

Co-authored-by: Matt Griswold <grizz@20c.com>

Ctl/VERSION

@@ -1 +1 @@
-3.2.0
+3.3.0-federation.0

docs/netbox_sync.md

@@ -0,0 +1,19 @@
+## Usage
+
+Once the organization has set up service federation for their NetBox instance
+the following command can be used to run the netbox sync job.
+
+```python
+Ctl/dev/run.sh ixctl_netbox_sync $ORG_SLUG $IX_SLUG
+```
+
+- `$ORG_SLUG` is the organization slug
+- `$IX_SLUG` is the IXP slug (optional, if not provided all IXPs will be synced)
+
+## Data pushed to NetBox
+
+IxCtl Source of Truth:
+
+- mac addresses
+- MTU
+- Prefixes (will also remove prefixes that are no longer in ixctl)

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "ixctl"
 repository = "https://github.com/fullctl/ixctl"
-version = "3.2.0"
+version = "3.3.0+federation.0"
 description = "ix control"
 authors = ["20C <code@20c.com>"]
 license = "Apache-2.0"
@@ -20,7 +20,7 @@ ixctl = "ixctl.cli:main"
 
 [tool.poetry.dependencies]
 python = "^3.9"
-fullctl = { git = "https://github.com/fullctl/fullctl.git", branch = "prep-release" }
+fullctl = { git = "https://github.com/fullctl/fullctl.git", branch = "federation" }
 arouteserver = ">=1.17"
 pydantic = ">=1.10.2"
 

src/django_ixctl/admin.py

@@ -5,6 +5,7 @@
 from django_ixctl.models import (
     InternetExchange,
     InternetExchangeMember,
+    InternetExchangePrefix,
     Network,
     PermissionRequest,
     Routeserver,
@@ -43,13 +44,20 @@ class RouteserverInline(BaseTabularAdmin):
     model = Routeserver
 
 
+class PrefixInline(admin.TabularInline):
+    model = InternetExchangePrefix
+    extra = 0
+
+
 @admin.register(InternetExchange)
 class InternetInternetExchangeAdmin(BaseAdmin):
     list_display = ("name", "id", "org", "source_of_truth")
     list_filter = ("source_of_truth",)
     readonly_fields = ("org",)
     search_fields = ("name", "instance__org__slug", "instance__org__name")
 
+    inlines = (PrefixInline,)
+
     def org(self, obj):
         return obj.instance.org
 

src/django_ixctl/management/commands/ixctl_netbox_sync.py

@@ -0,0 +1,29 @@
+from fullctl.django.management.commands.base import CommandInterface
+from fullctl.django.models.concrete import Organization
+from fullctl.service_bridge.context import ServiceBridgeContext
+
+import django_ixctl.sync.netbox as netbox
+
+
+class Command(CommandInterface):
+    help = "Pull netbox data for specified organization"
+
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+
+        parser.add_argument("org_slug", nargs="?")
+        # optional ix_slug argument
+        parser.add_argument("ix_slug", nargs="?")
+
+    def run(self, *args, **kwargs):
+        org_slug = kwargs.get("org_slug")
+        ix_slug = kwargs.get("ix_slug")
+        org = Organization.objects.get(slug=org_slug)
+        with ServiceBridgeContext(org):
+            self.log_info(f"Pushing updates to netbox for {org_slug}")
+
+            netbox.push(org, ix_slug=ix_slug)
+
+            # self.log_info(f"Pulling netbox data for {org_slug}")
+            # netbox.pull(org)
+            # self.log_info(f"Pulled netbox data for {org_slug}")

src/django_ixctl/migrations/0018_internetexchange_mtu_internetexchange_vlan_id_and_more.py

@@ -0,0 +1,80 @@
+# Generated by Django 4.2.9 on 2024-03-14 11:17
+
+import django.db.models.deletion
+import django.db.models.manager
+import django_handleref.models
+import netfields.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_ixctl", "0017_internetexchangemember_port"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="internetexchange",
+            name="mtu",
+            field=models.PositiveSmallIntegerField(
+                blank=True, help_text="MTU for the exchange", null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="internetexchange",
+            name="vlan_id",
+            field=models.PositiveIntegerField(
+                blank=True, help_text="VLAN ID for the exchange", null=True
+            ),
+        ),
+        migrations.CreateModel(
+            name="InternetExchangePrefix",
+            fields=[
+                ("id", models.AutoField(primary_key=True, serialize=False)),
+                (
+                    "created",
+                    django_handleref.models.CreatedDateTimeField(
+                        auto_now_add=True, verbose_name="Created"
+                    ),
+                ),
+                (
+                    "updated",
+                    django_handleref.models.UpdatedDateTimeField(
+                        auto_now=True, verbose_name="Updated"
+                    ),
+                ),
+                ("version", models.IntegerField(default=0)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("ok", "Ok"),
+                            ("pending", "Pending"),
+                            ("deactivated", "Deactivated"),
+                            ("failed", "Failed"),
+                            ("expired", "Expired"),
+                        ],
+                        default="ok",
+                        max_length=12,
+                    ),
+                ),
+                ("prefix", netfields.fields.CidrAddressField(max_length=43)),
+                (
+                    "ix",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="prefixes",
+                        to="django_ixctl.internetexchange",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Internet Exchange Prefix",
+                "verbose_name_plural": "Internet Exchange Prefixes",
+                "db_table": "ixctl_prefix",
+            },
+            managers=[
+                ("handleref", django.db.models.manager.Manager()),
+            ],
+        ),
+    ]

src/django_ixctl/migrations/0019_alter_internetexchangeprefix_unique_together.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.2.9 on 2024-03-14 12:46
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_ixctl", "0018_internetexchange_mtu_internetexchange_vlan_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="internetexchangeprefix",
+            unique_together={("prefix", "ix")},
+        ),
+    ]

src/django_ixctl/models/ixctl.py

@@ -34,7 +34,7 @@
 from fullctl.django.inet.validators import validate_as_set
 from fullctl.django.models.abstract.base import HandleRefModel, PdbRefModel
 from fullctl.django.models.concrete import Instance, Organization
-from netfields import InetAddressField, MACAddressField
+from netfields import CidrAddressField, InetAddressField, MACAddressField
 
 import django_ixctl.enum
 import django_ixctl.models.tasks
@@ -116,6 +116,18 @@ class InternetExchange(PdbRefModel):
         ),
     )
 
+    mtu = models.PositiveSmallIntegerField(
+        null=True,
+        blank=True,
+        help_text=_("MTU for the exchange"),
+    )
+
+    vlan_id = models.PositiveIntegerField(
+        null=True,
+        blank=True,
+        help_text=_("VLAN ID for the exchange"),
+    )
+
     # TODO: use service-bridge reference field?
     # this field is already defined through PdbRefModel, however we need to
     # override it here to give it a help-text
@@ -168,13 +180,23 @@ def create_from_pdb(cls, instance, pdb_object, save=True, **fields):
 
         ix.name = pdb_object.name
         ix.slug = cls.default_slug(ix.name)
+        ix.mtu = pdb_object.mtu
 
         if save:
             ix.save()
 
+        # create members from pdb
+
         for netixlan in pdbctl.NetworkIXLan().objects(ix=pdb_object.id, join="net"):
             InternetExchangeMember.create_from_pdb(netixlan, ix=ix)
 
+        # create prefixes from pdb
+
+        pdb_prefixes = pdbctl.IXLanPrefix().objects(ix=pdb_object.id)
+
+        for prefix in pdb_prefixes:
+            InternetExchangePrefix.objects.create(ix=ix, prefix=prefix.prefix)
+
         return ix
 
     @classmethod
@@ -266,6 +288,40 @@ def __str__(self):
         return f"{self.name} ({self.id})"
 
 
+@grainy_model(
+    namespace="ix",
+    namespace_instance="ix.{instance.org.permission_id}.{instance.ix_id}.prefix",
+)
+class InternetExchangePrefix(HandleRefModel):
+    """
+    Describes a CIDR prefix for an internet exchange
+    """
+
+    ix = models.ForeignKey(
+        InternetExchange,
+        related_name="prefixes",
+        on_delete=models.CASCADE,
+    )
+
+    prefix = CidrAddressField()
+
+    class Meta:
+        db_table = "ixctl_prefix"
+        verbose_name = _("Internet Exchange Prefix")
+        verbose_name_plural = _("Internet Exchange Prefixes")
+        unique_together = (("prefix", "ix"),)
+
+    class HandleRef:
+        tag = "prefix"
+
+    @property
+    def org(self):
+        return self.ix.instance.org
+
+    def __str__(self):
+        return f"{self.prefix} - {self.ix.name}"
+
+
 class OrganizationDefaultExchange(models.Model):
     """
     Describes the default exchange for an organization

src/django_ixctl/rest/serializers/ixctl.py

@@ -108,6 +108,26 @@ class Meta:
         fields = ["user", "type", "org"]
 
 
+@register
+class Prefix(ModelSerializer):
+    class Meta:
+        model = models.InternetExchangePrefix
+        fields = ["id", "ix", "prefix"]
+
+
+@register
+class UpdateLAN(ModelSerializer):
+    ref_tag = "update_lan"
+
+    class Meta:
+        model = models.InternetExchange
+        fields = [
+            "id",
+            "vlan_id",
+            "mtu",
+        ]
+
+
 @register
 class InternetExchange(ModelSerializer):
     slug = serializers.SlugField(required=False)
@@ -123,6 +143,8 @@ class Meta:
             "slug",
             "source_of_truth",
             "verified",
+            "mtu",
+            "vlan_id",
         ]
 
         read_only_fields = [
@@ -135,7 +157,6 @@ def validate(self, cleaned_data):
         slug = cleaned_data.get("slug")
         if not slug:
             slug = models.InternetExchange.default_slug(cleaned_data["name"])
-        print("checking", instance, slug)
         qset = models.InternetExchange.objects.filter(instance=instance, slug=slug)
         if qset.exists():
             raise ValidationError(

src/django_ixctl/rest/serializers/service_bridge.py

@@ -7,6 +7,13 @@
 Serializers, register = serializer_registry()
 
 
+@register
+class Prefix(ModelSerializer):
+    class Meta:
+        model = models.InternetExchangePrefix
+        fields = ["id", "ix", "prefix"]
+
+
 @register
 class InternetExchange(ModelSerializer):
     org_id = serializers.SerializerMethodField()
@@ -19,7 +26,10 @@ class Meta:
             "slug",
             "pdb_id",
             "name",
+            "slug",
             "verified",
+            "mtu",
+            "vlan_id",
         ]
 
     def get_org_id(self, ix):