Skip to content

Commit

Permalink
Dockerfile/project updates
Browse files Browse the repository at this point in the history 
Signed-off-by: Clemens Vasters <clemens@vasters.com>
  • Loading branch information
@clemensv
clemensv committed Sep 17, 2024
1 parent a32b36d commit 209b57a
Show file tree
Hide file tree
Showing 30 changed files with 303 additions and 160 deletions.
4 changes: 1 addition & 3 deletions gtfs/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ COPY . /app
RUN apt-get update && apt-get install -y git

# Install the required Python packages
RUN pip install gtfs_rt_producer/gtfs_rt_producer_data
RUN pip install gtfs_rt_producer/gtfs_rt_producer_kafka_producer
RUN pip install gtfs_rt_bridge/.
RUN pip install .

# Define environment variables (default values)

Expand Down
4 changes: 2 additions & 2 deletions gtfs/generate-template.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ $templateResources = @(
"image" = "[parameters('imageName')]"
"resources" = @{
"requests" = @{
"cpu" = 0.5
"memoryInGB" = 1
"cpu" = 1
"memoryInGB" = 2
}
}
"environmentVariables" = @(
Expand Down
6 changes: 3 additions & 3 deletions gtfs/gtfs_rt_bridge/pyproject.toml → gtfs/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ description = "GTFS-RT Bridge"
authors = ["Clemens Vasters"]
readme = "README.md"
version = "0.1.0" # Placeholder version, dynamic versioning can be handled with plugins
packages = [{include = "gtfs_rt_bridge", from = "src"}]
packages = [{include = "gtfs_rt_bridge", from = "gtfs_rt_bridge/src"}]

[tool.poetry.dependencies]
python = "<4.0,>=3.10"
gtfs_rt-producer-kafka-producer = "*"
gtfs_rt-producer-data = "*"
gtfs-rt-producer-data = {path = "gtfs_rt_producer/gtfs_rt_producer_data"}
gtfs-rt-producer-kafka-producer = {path = "gtfs_rt_producer/gtfs_rt_producer_kafka_producer"}
confluent-kafka = ">=2.5.3"
cloudevents = ">=1.11.0"
gtfs-realtime-bindings = ">=1.0.0"
Expand Down
2 changes: 1 addition & 1 deletion noaa/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ WORKDIR /app
COPY . /app

# Install the required Python packages
RUN pip install --no-cache-dir requests confluent-kafka cloudevents dataclasses_json numpy
RUN pip install .

# Make port 80 available to the world outside this container
EXPOSE 80
Expand Down
3 changes: 2 additions & 1 deletion noaa/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@ description = "A project to fetch data from NOAA API"
authors = ["Clemens Vasters <clemensv@microsoft.com>"]

[tool.poetry.dependencies]

python = ">=3.8"
requests = ">=2.32.3"
confluent-kafka = ">=2.5.3"
cloudevents = ">=1.11.0"
dataclasses_json = ">=0.6.7"
numpy = ">=2.1.1"


[tool.poetry.dev-dependencies]
pytest = ">=8.3.2"
Expand Down
2 changes: 0 additions & 2 deletions pegelonline/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ WORKDIR /app
COPY . /app

# Install the required Python packages
RUN pip install pegelonline_producer/pegelonline_producer_data
RUN pip install pegelonline_producer/pegelonline_producer_kafka_producer
RUN pip install .

# Define environment variables (default values)
Expand Down
4 changes: 3 additions & 1 deletion pegelonline/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ description = "A project to fetch data from PegelOnline API"
authors = ["Clemens Vasters <clemensv@microsoft.com>"]

[tool.poetry.dependencies]
pegelonline_producer_data = {path = "pegelonline_producer/pegelonline_producer_data"}
pegelonline_producer_kafka_producer = {path = "pegelonline_producer/pegelonline_producer_kafka_producer"}
python = ">=3.8"
requests = ">=2.32.3"
confluent-kafka = ">=2.5.3"
cloudevents = ">=1.11.0"
dataclasses_json = ">=0.6.7"
numpy = ">=2.1.1"


[tool.poetry.dev-dependencies]
pytest = ">=8.3.3"
Expand Down
2 changes: 0 additions & 2 deletions rss/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ WORKDIR /app
COPY . /app

# Install the required Python packages
RUN pip install rssbridge_producer/rssbridge_producer_data
RUN pip install rssbridge_producer/rssbridge_producer_kafka_producer
RUN pip install .

# Define environment variables (default values)
Expand Down
73 changes: 67 additions & 6 deletions rss/azure-template.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,62 @@
{
"contentVersion": "1.0.0.0",
"resources": [
{
"location": "[resourceGroup().location]",
"apiVersion": "2021-04-01",
"kind": "StorageV2",
"properties": {
"accessTier": "Hot"
},
"name": "[variables('storageAccountName')]",
"type": "Microsoft.Storage/storageAccounts",
"sku": {
"name": "Standard_LRS"
}
},
{
"location": "[resourceGroup().location]",
"apiVersion": "2021-04-01",
"dependsOn": [
"[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]"
],
"properties": {
"protocolSettings": {
"smb": {
"enabled": true
}
}
},
"name": "[concat(variables('storageAccountName'), '/default/')]",
"type": "Microsoft.Storage/storageAccounts/fileServices"
},
{
"location": "[resourceGroup().location]",
"apiVersion": "2021-04-01",
"dependsOn": [
"[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]"
],
"properties": {
"shareQuota": 5120
},
"name": "[concat(variables('storageAccountName'), '/default/', variables('fileShareName'))]",
"type": "Microsoft.Storage/storageAccounts/fileServices/shares"
},
{
"apiVersion": "2021-09-01",
"type": "Microsoft.ContainerInstance/containerGroups",
"properties": {
"restartPolicy": "Always",
"volumes": [
{
"azureFile": {
"shareName": "[variables('fileShareName')]",
"storageAccountName": "[variables('storageAccountName')]",
"storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2019-06-01').keys[0].value]"
},
"name": "azurefilevolume"
}
],
"osType": "Linux",
"containers": [
{
Expand All @@ -16,6 +67,12 @@
"memoryInGB": 1
}
},
"volumeMounts": [
{
"name": "azurefilevolume",
"mountPath": "/mnt/fileshare"
}
],
"image": "[parameters('imageName')]",
"environmentVariables": [
{
Expand All @@ -30,6 +87,10 @@
"name": "FEED_URLS",
"value": "[parameters('feedUrls')]"
}
{
"name": "STATE_DIR",
"value": "/mnt/fileshare/state"
}
]
},
"name": "[parameters('appName')]"
Expand All @@ -51,21 +112,21 @@
"logAnalyticsWorkspaceId": {
"type": "string",
"metadata": {
"description": "The Id of the Log Analytics workspace. Leave empty to skip log analytics configuration."
"description": "The Id of the Log Analytics workspace. In the portal, you find this under Settings -> Agents -> Windows/Linux Servers -> Agent Instructions."
},
"defaultValue": ""
"nullable": false
},
"logAnalyticsWorkspaceKey": {
"type": "securestring",
"metadata": {
"description": "The key of the Log Analytics workspace. Leave empty to skip log analytics configuration."
"description": "The primary or secondary key of the Log Analytics workspace. In the portal, you find this under Settings -> Agents -> Windows/Linux Servers -> Agent Instructions."
},
"defaultValue": ""
"nullable": false
},
"connectionStringSecret": {
"type": "securestring",
"metadata": {
"description": "The Microsoft Fabric Event Stream custom input endpoint connection string."
"description": "The Microsoft Fabric Event Stream custom input endpoint or Azure Event Hubs connection string."
}
},
"feedUrls": {
Expand All @@ -81,7 +142,7 @@
"metadata": {
"description": "The name of the container instance."
},
"defaultValue": "[resourceGroup().name]"
"defaultValue": "[if(resourceGroup().name, resourceGroup().name, 'rss-bridge')]"
},
"imageName": {
"type": "string",
Expand Down
71 changes: 63 additions & 8 deletions rss/generate-template.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ $templateParameters = @{
"connectionStringSecret" = @{
"type" = "securestring"
"metadata" = @{
"description" = "The Microsoft Fabric Event Stream custom input endpoint connection string."
"description" = "The Microsoft Fabric Event Stream custom input endpoint or Azure Event Hubs connection string."
}
}
"feedUrls" = @{
Expand All @@ -20,7 +20,7 @@ $templateParameters = @{
}
"appName" = @{
"type" = "string"
"defaultValue" = "[resourceGroup().name]"
"defaultValue" = "[if(resourceGroup().name, resourceGroup().name, 'rss-bridge')]"
"metadata" = @{
"description" = "The name of the container instance."
}
Expand All @@ -33,26 +33,66 @@ $templateParameters = @{
"description" = "The name of the container image."
}
}
# New parameter for Log Analytics workspace name
"logAnalyticsWorkspaceId" = @{
"type" = "string"
"defaultValue" = ""
"nullable" = $false
"metadata" = @{
"description" = "The Id of the Log Analytics workspace. Leave empty to skip log analytics configuration."
"description" = "The Id of the Log Analytics workspace. In the portal, you find this under Settings -> Agents -> Windows/Linux Servers -> Agent Instructions."
}
}
"logAnalyticsWorkspaceKey" = @{
"type" = "securestring"
"defaultValue" = ""
"nullable" = $false
"metadata" = @{
"description" = "The key of the Log Analytics workspace. Leave empty to skip log analytics configuration."
"description" = "The primary or secondary key of the Log Analytics workspace. In the portal, you find this under Settings -> Agents -> Windows/Linux Servers -> Agent Instructions."
}
}
}
"variables" = @{
"storageAccountName" = "[concat(replace(parameters('appName'), '-', ''), 'stg')]"
"fileShareName" = "fileshare"
}
}

# Define resources in the ARM template
$templateResources = @(
@{
"type" = "Microsoft.Storage/storageAccounts"
"apiVersion" = "2021-04-01"
"name" = "[variables('storageAccountName')]"
"location" = "[resourceGroup().location]"
"sku" = @{
"name" = "Standard_LRS"
}
"kind" = "StorageV2"
"properties" = @{
"accessTier" = "Hot"
}
}
@{
"type" = "Microsoft.Storage/storageAccounts/fileServices"
"apiVersion" = "2021-04-01"
"name" = "[concat(variables('storageAccountName'), '/default/')]"
"location" = "[resourceGroup().location]"
"dependsOn" = @("[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]")
"properties" = @{
"protocolSettings" = @{
"smb" = @{
"enabled" = $true
}
}
}
}
@{
"type" = "Microsoft.Storage/storageAccounts/fileServices/shares"
"apiVersion" = "2021-04-01"
"name" = "[concat(variables('storageAccountName'), '/default/', variables('fileShareName'))]"
"location" = "[resourceGroup().location]"
"dependsOn" = @("[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]")
"properties" = @{
"shareQuota" = 5120
}
}
@{
"type" = "Microsoft.ContainerInstance/containerGroups"
"apiVersion" = "2021-09-01"
Expand Down Expand Up @@ -84,12 +124,27 @@ $templateResources = @(
"value" = "[parameters('feedUrls')]"
}
)
"volumeMounts" = @(
@{
"name" = "azurefilevolume"
"mountPath" = "/mnt/fileshare"
}
)
}
}
)
"osType" = "Linux"
"restartPolicy" = "Always"
# Conditional diagnostics section using here-string
"volumes" = @(
@{
"name" = "azurefilevolume"
"azureFile" = @{
"shareName" = "[variables('fileShareName')]"
"storageAccountName" = "[variables('storageAccountName')]"
"storageAccountKey" = "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2019-06-01').keys[0].value]"
}
}
)
"diagnostics" = @{
"logAnalytics" = @{
"workspaceId" = "[parameters('logAnalyticsWorkspaceId')]"
Expand Down
2 changes: 2 additions & 0 deletions rss/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ authors = ["Clemens Vasters <clemensv@microsoft.com>"]


[tool.poetry.dependencies]
rssbridge_producer_data = {path = "rssbridge_producer/rssbridge_producer_data"}
rssbridge_producer_kafka_producer = {path = "rssbridge_producer/rssbridge_producer_kafka_producer"}
python = ">=3.8"
requests = ">=2.32.3"
confluent-kafka = ">=2.5.3"
Expand Down
Loading

0 comments on commit 209b57a

Please sign in to comment.