` fails """
+
+ # Create DB entry in TrustUpdates table
+ trust_object = await models.TrustUpdates.create(recipe_id=recipe_id)
+
+ # Post Slack Message
+ await api.send_msg.trust_diff_msg(diff_msg, trust_object)
+
+ # Mark the recipe disabled
+ recipe_object = await models.Recipes.filter(recipe_id=trust_object.recipe_id).first()
+ recipe_object.enabled = False
+ await recipe_object.save()
+ return { "result": "Success" }
diff --git a/db/__init__.py b/pkgbot/api/slack/__init__.py
similarity index 100%
rename from db/__init__.py
rename to pkgbot/api/slack/__init__.py
diff --git a/api/slack/block_builders.py b/pkgbot/api/slack/block_builders.py
similarity index 54%
rename from api/slack/block_builders.py
rename to pkgbot/api/slack/block_builders.py
index 34a3178..df98d51 100644
--- a/api/slack/block_builders.py
+++ b/pkgbot/api/slack/block_builders.py
@@ -1,11 +1,16 @@
-#!/usr/local/autopkg/python
-
from fastapi import Depends
-import utils
-from db import models
+from pkgbot import config
+from pkgbot.utilities import common as utility
+from pkgbot.db import models
+
+
+log = utility.log
+config = config.load_config()
+
+secure = "s" if config.PkgBot.get("enable_ssl") else ""
+pkgbot_server = f"http{secure}://{config.PkgBot.get('host')}:{config.PkgBot.get('port')}"
-log = utils.log
async def brick_header(pkg_object: models.Package_In = Depends(models.Package_In)):
@@ -24,17 +29,12 @@ async def brick_main(pkg_object: models.Package_In = Depends(models.Package_In))
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "*Name:* `{}`\n*Version:* `{}`\n*Package Name:* `{}`".format(
- pkg_object.dict().get("name"), pkg_object.dict().get("version"),
- pkg_object.dict().get("pkg_name", "Unknown")
- )
+ "text": f"*Name:* `{pkg_object.dict().get('name')}`\n*Version:* `{pkg_object.dict().get('version')}`\n*Package Name:* `{pkg_object.dict().get('pkg_name', 'Unknown')}`"
},
"accessory": {
"type": "image",
- "image_url": "{}/iconservlet/?id={}".format(pkg_object.dict().get("jps_url"),
- pkg_object.dict().get("icon_id")
- ),
- "alt_text": "computer thumbnail"
+ "image_url": f"{pkgbot_server}/static/icons/{pkg_object.dict().get('icon')}",
+ "alt_text": ":new:"
}
}
@@ -46,8 +46,7 @@ async def brick_footer_dev(pkg_object: models.Package_In = Depends(models.Packag
"elements": [
{
"type": "mrkdwn",
- "text": "*Dev*: {}\t*Uploaded by*: @{}".format(
- pkg_object.dict().get("packaged_date"), "PkgBot")
+ "text": f"*Dev*: {pkg_object.dict().get('packaged_date')}\t*Uploaded by*: @{config.Slack.get('bot_name')}"
}
]
}
@@ -57,9 +56,7 @@ async def brick_footer_promote(pkg_object: models.Package_In = Depends(models.Pa
return {
"type": "mrkdwn",
- "text": "*Prod*: {}\t*Approved by*: @{}".format(
- pkg_object.dict().get("promoted_date"), pkg_object.dict().get("status_updated_by")
- )
+ "text": f"*Prod*: {pkg_object.dict().get('promoted_date')}\t*Approved by*: @{pkg_object.dict().get('status_updated_by')}"
}
@@ -67,21 +64,18 @@ async def brick_footer_denied(pkg_object: models.Package_In = Depends(models.Pac
return {
"type": "mrkdwn",
- "text": "*Denied by*: @{}\t*On*: @{}".format(
- pkg_object.dict().get("status_updated_by"), pkg_object.dict().get("last_update"))
+ "text": f"*Denied by*: @{pkg_object.dict().get('status_updated_by')}\t*On*: {pkg_object.dict().get('last_update')}"
}
-async def brick_footer_denied_trust(error_object):
+async def brick_footer_denied_trust(trust_object):
return {
"type": "context",
"elements": [
{
"type": "mrkdwn",
- "text": "*Denied by*: @{}\t*On*: {}".format(
- error_object.dict().get("status_updated_by"),
- error_object.dict().get("last_update"))
+ "text": f"*Denied by*: @{trust_object.dict().get('status_updated_by')}\t*On*: {trust_object.dict().get('last_update')}"
}
]
}
@@ -89,7 +83,7 @@ async def brick_footer_denied_trust(error_object):
async def brick_button(pkg_object: models.Package_In = Depends(models.Package_In)):
- return (
+ return (
{
"type": "section",
"text": {
@@ -108,7 +102,7 @@ async def brick_button(pkg_object: models.Package_In = Depends(models.Package_In
"text": "Approve"
},
"style": "primary",
- "value": "Package:{}".format(pkg_object.dict().get("id"))
+ "value": f"Package:{pkg_object.dict().get('id')}"
},
{
"type": "button",
@@ -118,7 +112,7 @@ async def brick_button(pkg_object: models.Package_In = Depends(models.Package_In
"text": "Deny"
},
"style": "danger",
- "value": "Package:{}".format(pkg_object.dict().get("id"))
+ "value": f"Package:{pkg_object.dict().get('id')}"
}
]
}
@@ -131,7 +125,7 @@ async def brick_error(recipe_id, error):
"type": "header",
"text": {
"type": "plain_text",
- "text": "Encountered an error in: {}".format(recipe_id),
+ "text": f"Encountered an error in: {recipe_id}",
"emoji": True
}
},
@@ -139,51 +133,64 @@ async def brick_error(recipe_id, error):
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "```{}```".format(error)
+ "text": f"```{error}```",
+ "verbatim": True
},
"accessory": {
"type": "image",
- "image_url": "computer thumbnail",
+ "image_url": f"{pkgbot_server}/static/icons/{config.PkgBot.get('icon_error')}",
"alt_text": ":x:"
}
+ },
+ {
+ "type": "actions",
+ "elements": [
+ {
+ "type": "button",
+ "text": {
+ "type": "plain_text",
+ "emoji": True,
+ "text": "Acknowledge"
+ },
+ "style": "danger",
+ "value": "Error:ack"
+ }
+ ]
}]
-async def brick_update_trust_success_msg(error_object):
+async def brick_update_trust_success_msg(trust_object):
return {
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "Trust info was updated for: `{}`".format(
- error_object.dict().get("recipe_id"))
+ "text": f"Trust info was updated for: `{trust_object.dict().get('recipe_id')}`",
+ "verbatim": True
}
}
-async def brick_footer_update_trust_success_msg(error_object):
+async def brick_footer_update_trust_success_msg(trust_object):
return {
"type": "context",
"elements": [
{
"type": "mrkdwn",
- "text": "*Updated by*: @{}\t*On*: {}".format(
- error_object.dict().get("status_updated_by"),
- error_object.dict().get("last_update"))
+ "text": f"*Updated by*: @{trust_object.dict().get('status_updated_by')}\t*On*: {trust_object.dict().get('last_update')}"
}
]
}
-async def brick_update_trust_error_msg(error_object, msg):
+async def brick_update_trust_error_msg(trust_object, msg):
return [{
"type": "header",
"text": {
"type": "plain_text",
- "text": "Failed to update trust info for `{}`".format(
- error_object.dict().get("recipe_id")),
+ "text": f"Failed to update trust info for `{trust_object.dict().get('recipe_id')}`",
"emoji": True
}
},
@@ -191,11 +198,11 @@ async def brick_update_trust_error_msg(error_object, msg):
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "```{}```".format(msg)
+ "text": f"```{msg}```"
},
"accessory": {
"type": "image",
- "image_url": "computer thumbnail",
+ "image_url": f"{pkgbot_server}/static/icons/{config.PkgBot.get('icon_error')}",
"alt_text": ":x:"
}
}]
@@ -212,20 +219,20 @@ async def brick_deny_pkg(pkg_object: models.Package_In = Depends(models.Package_
}
-async def brick_deny_trust(error_object):
+async def brick_deny_trust(trust_object):
return {
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "Denied update to trust info for `{}`".format(
- error_object.dict().get("recipe_id"))
+ "text": f"Denied update to trust info for `{trust_object.dict().get('recipe_id')}`",
+ "verbatim": True
},
- # "accessory": {
- # "type": "image",
- # "image_url": "computer thumbnail",
- # "alt_text": ":x:"
- # }
+ "accessory": {
+ "type": "image",
+ "image_url": f"{pkgbot_server}/static/icons/{config.PkgBot.get('icon_denied')}",
+ "alt_text": ":denied:"
+ }
}
@@ -246,13 +253,14 @@ async def brick_trust_diff_main(recipe):
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "*Recipe:* `{}`\n\n_Trust diff review required._\n\n".format(recipe)
+ "text": f"*Recipe:* `{recipe}`\n\n_Trust diff review required._\n\n",
+ "verbatim": True
},
- # "accessory": {
- # "type": "image",
- # "image_url": "computer thumbnail",
- # "alt_text": ":x:"
- # }
+ "accessory": {
+ "type": "image",
+ "image_url": f"{pkgbot_server}/static/icons/{config.PkgBot.get('icon_warning')}",
+ "alt_text": ":warning:"
+ }
}
@@ -262,7 +270,8 @@ async def brick_trust_diff_content(error):
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "*Diff Output:*```{}```".format(error)
+ "text": f"*Diff Output:*```{error}```",
+ "verbatim": True
}
}
@@ -280,7 +289,7 @@ async def brick_trust_diff_button(id):
"text": "Approve"
},
"style": "primary",
- "value": "Trust:{}".format(id)
+ "value": f"Trust:{id}"
},
{
"type": "button",
@@ -290,7 +299,7 @@ async def brick_trust_diff_button(id):
"text": "Deny"
},
"style": "danger",
- "value": "Trust:{}".format(id)
+ "value": f"Trust:{id}"
}
]
}
@@ -311,13 +320,13 @@ async def unauthorized(user):
"text": {
"type": "mrkdwn",
"text": "_*Warning:*_ you are not a PkgBot admin and are not authorized to "
- "perform this action.\n\n`{}` will be reported to the "
- "robot overloads.".format(user)
+ f"perform this action.\n\n`{user}` will be reported to the "
+ "robot overloads."
},
"accessory": {
"type": "image",
- "image_url": "https://as1.ftcdn.net/jpg/01/81/82/24/500_F_181822453_iQYjSxsW1AXa8FHOA6ecgdZEmrBdfInD.jpg",
- "alt_text": ":x:"
+ "image_url": f"{pkgbot_server}/static/icons/{config.PkgBot.get('icon_permission_denied')}",
+ "alt_text": ":denied:"
}
}
]
@@ -330,11 +339,12 @@ async def missing_recipe_msg(recipe_id, text):
"type": "section",
"text": {
"type": "mrkdwn",
- "text": "Failed to {} `{}`".format(text, recipe_id),
+ "text": f"Failed to {text} `{recipe_id}`",
+ "verbatim": True
},
"accessory": {
"type": "image",
- "image_url": "error",
+ "image_url": f"{pkgbot_server}/static/icons/{config.PkgBot.get('icon_error')}",
"alt_text": ":x:"
}
}
diff --git a/pkgbot/api/slack/bot.py b/pkgbot/api/slack/bot.py
new file mode 100644
index 0000000..8659600
--- /dev/null
+++ b/pkgbot/api/slack/bot.py
@@ -0,0 +1,406 @@
+import hmac
+import json
+import ssl
+import time
+import certifi
+
+from fastapi import APIRouter, Depends, Request
+
+from slack_sdk.errors import SlackApiError
+from slack_sdk.web.async_client import AsyncWebClient
+from slack_sdk.webhook.async_client import AsyncWebhookClient
+
+from pkgbot import api, config, settings
+from pkgbot.db import models
+from pkgbot.utilities import common as utility
+
+
+config = config.load_config()
+log = utility.log
+
+SlackBot = None
+ssl_context = ssl.create_default_context(cafile=certifi.where())
+router = APIRouter(
+ prefix = "/slackbot",
+ tags = ["slackbot"],
+ responses = settings.api.custom_responses
+)
+
+
+class SlackClient(object):
+
+ def __init__(self, **kwargs):
+ self.token = kwargs["token"]
+ self.bot_name = kwargs["bot_name"]
+ self.channel = kwargs["channel"]
+ self.slack_id = kwargs["slack_id"]
+
+ self.client = AsyncWebClient(token=self.token, ssl=ssl_context)
+
+
+ async def post_message(self, blocks: str, text: str = "Pkg status incoming..."):
+
+ try:
+ return await self.client.chat_postMessage(
+ channel = self.channel,
+ text = text,
+ blocks = blocks,
+ username = self.bot_name
+ )
+
+ except SlackApiError as error:
+ log.error(f"Failed to post message: {error.response['error']}\n{error}")
+ return { "Failed to post message": error.response["error"] }
+
+
+ async def update_message(self, blocks: str, ts: str, text: str = "Updated message..."):
+
+ try:
+ return await self.client.chat_update(
+ channel = self.channel,
+ text = text,
+ blocks = blocks,
+ ts = ts
+ )
+
+ except SlackApiError as error:
+ log.error(f"Failed to update {ts}: {error.response['error']}\n{error}")
+ return { f"Failed to update {ts}": error.response["error"] }
+
+
+ async def delete_message(self, ts: str):
+
+ try:
+ await self.client.chat_delete(channel=self.channel, ts=ts)
+ return { "result": "Successfully deleted message" }
+
+ except SlackApiError as error:
+ log.error(f"Failed to delete {ts}: {error.response['error']}\n{error}")
+ return { f"Failed to delete {ts}": error.response["error"] }
+
+
+ async def update_message_with_response_url(
+ self, response_url: str, blocks: str, text: str = "Pkg status update..."):
+
+ try:
+ webhook = AsyncWebhookClient(url=response_url, ssl=ssl_context)
+ response = await webhook.send(
+ text = text,
+ blocks = blocks,
+ replace_original = True
+ )
+
+ if response.status_code != 200:
+ log.error(
+ f"Failed to update message! Status code: {response.status_code} | Error message: {response.body}")
+ else:
+ log.debug("Successfully updated msg via response_url")
+
+ return response
+
+ except SlackApiError as error:
+ log.error(
+ f"Failed to update {response_url}\nFull Error:\n{error}\nerror.dir: {dir(error)}\nerror.response['error']: {error.response['error']}")
+ return { f"Failed to update {response_url}": error.response["error"] }
+
+
+ async def post_ephemeral_message(
+ self, user: str, blocks: str, channel: str = None, text: str = "Private Note"):
+
+ try:
+ return await self.client.chat_postEphemeral(
+ channel = channel or self.channel,
+ user = user,
+ text = text,
+ blocks = blocks,
+ username = self.bot_name
+ )
+
+ except SlackApiError as error:
+ log.error(
+ f"Failed to post ephemeral message: {error.response['error']}\nFull Error:\n{error}")
+ return { "Failed to post ephemeral message": error.response["error"] }
+
+
+ async def file_upload(self, content=None, file=None, filename=None, filetype=None,
+ title=None, text=None, thread_ts=None):
+
+ try:
+ return await self.client.files_upload(
+ channels = self.channel,
+ content = content,
+ file = file,
+ filename = filename,
+ filetype = filetype,
+ title = title,
+ initial_comment = text,
+ thread_ts = thread_ts,
+ username = self.bot_name
+ )
+
+ except SlackApiError as error:
+ log.error(f"Failed to upload {file}: {error.response['error']}\nFull Error:\n{error}")
+ return { f"Failed to upload {file}": error.response["error"] }
+
+
+ async def invoke_reaction(self, **kwargs):
+
+ kwargs |= {
+ "channel": kwargs.get("channel", self.channel),
+ "timestamp": str(kwargs.get("ts"))
+ }
+
+ if "ts" in kwargs:
+ del kwargs["ts"]
+
+ try:
+ return await self.client.api_call(
+ f"reactions.{kwargs.get('action')}",
+ params = kwargs
+ )
+
+ except SlackApiError as error:
+ error_key = error.response["error"]
+
+ if not (
+ kwargs.get("action") == "add" and error_key == "already_reacted" or
+ kwargs.get("action") == "remove" and error_key == "no_reaction"
+ ):
+ result = { f"Failed to invoke reaction on {kwargs.get('timestamp')}": error_key }
+ log.error(result)
+ return result
+
+ else:
+ log.debug("Unable to perform the specified reaction action")
+
+
+ async def reaction(self, action: str = None, emoji: str = None, ts: str = None, **kwargs):
+
+ # log.debug("Args:\n\taction: {}\n\temoji: {}\n\tts: {}\n\tkwargs: {}".format(
+ # action, emoji, ts, kwargs))
+
+ # log.debug("Checking current reactions")
+
+ # Force checking if this works or not.....
+ # It's not....
+ # response = await self.client.api_call(
+ # "reactions.get",
+ # http_verb = "GET",
+ # params = {
+ # 'channel': 'C0266ANUEJZ',
+ # 'timestamp': '1646121180.754269'
+ # }
+ # )
+
+##### This is currently not working....
+ # # Check if reaction exists or not...
+ # response = await self.invoke_reaction(action="get", ts=ts, http_verb="GET")
+ # # log.debug("forced get response:\n{}".format(response))
+ # reactions = response.get("message").get("reactions")
+
+ # for reaction in reactions:
+ # if (
+ # reaction.get("name") == kwargs.get("emoji") and
+ # elf.slack_id in reaction.get("users")
+ # ):
+ # log.debug("Reaction already exists")
+ # exists = True
+ # break
+
+ # log.debug("Reaction doesn't exist")
+ # exists = False
+
+ # if (
+ # action == "add" and exists == False or
+ # action == "remove" and exists == True
+ # ):
+
+ return await self.invoke_reaction(action=action, name=emoji, ts=ts, **kwargs)
+
+
+async def validate_slack_request(request: Request):
+
+ try:
+ slack_timestamp = request.headers.get("X-Slack-Request-Timestamp")
+
+ if abs(time.time() - int(slack_timestamp)) > 60 * 5:
+ # The request timestamp is more than five minutes from local time.
+ # It could be a replay attack, so let's ignore it.
+ return False
+
+ slack_body = (await request.body()).decode("UTF-8")
+ signature_basestring = (f"v0:{slack_timestamp}:{slack_body}").encode()
+
+ computed_signature = "v0=" + await utility.compute_hex_digest(
+ bytes(config.Slack.get("signing_secret"), "UTF-8"),
+ signature_basestring)
+
+ slack_signature = request.headers.get("X-Slack-Signature")
+
+ if hmac.compare_digest(computed_signature, slack_signature):
+ log.debug("Valid Slack message")
+ return True
+
+ else:
+ log.warning("Invalid Slack message!")
+ return False
+
+ except:
+
+ return False
+
+
+@router.on_event("startup")
+async def startup_constructor():
+
+ global SlackBot
+
+ SlackBot = SlackClient(
+ token = config.Slack.get("bot_token"),
+ bot_name = config.Slack.get("bot_name"),
+ channel = config.Slack.get("channel"),
+ slack_id = config.Slack.get("slack_id")
+ )
+
+
+@router.delete("/ts/{ts}", summary="Delete Slack message by timestamp",
+ description="Delete a Slack message by its timestamp.",
+ dependencies=[Depends(api.user.verify_admin)])
+async def delete_slack_message(timestamps: str | list):
+
+ if isinstance(timestamps, str):
+ timestamps = [timestamps]
+
+ results = {}
+
+ for ts in timestamps:
+ result = await SlackBot.delete_message(str(ts))
+
+ try:
+ results[ts] = result.response['error']
+ except Exception:
+ results[ts] = "Successfully deleted message"
+
+ return results
+
+
+@router.post("/receive", summary="Handles incoming messages from Slack",
+ description="This endpoint receives incoming messages from Slack and calls the required "
+ "actions based on the message after verifying the authenticity of the source.")
+async def receive(request: Request):
+
+ if await validate_slack_request(request):
+
+ form_data = await request.form()
+ payload = form_data.get("payload")
+ payload_object = json.loads(payload)
+
+ user_id = payload_object.get("user").get("id")
+ username = payload_object.get("user").get("username")
+ channel = payload_object.get("channel").get("id")
+ message_ts = payload_object.get("message").get("ts")
+ response_url = payload_object.get("response_url")
+
+ button_text = payload_object.get("actions")[0].get("text").get("text")
+ button_value_type, button_value = (
+ payload_object.get("actions")[0].get("value")).split(":")
+
+ # log.debug("Incoming details:\n"
+ # f"user id: {user_id}\nusername: {username}\nchannel: {channel}\nmessage_ts: "
+ # f"{message_ts}\nresponse_url: {response_url}\nbutton_text: {button_text}\n"
+ # f"button_value_type: {button_value_type}\nbutton_value: {button_value}\n"
+ # )
+
+ slack_user_object = models.PkgBotAdmin_In(
+ username = username,
+ slack_id = user_id
+ )
+
+ user_that_clicked = await api.user.get_user(slack_user_object)
+
+##### Disabled for testing
+##### Actually don't think this is needed........
+ # try:
+ # if user_that_clicked.full_admin:
+ # full_admin = True
+
+ # except:
+ # full_admin = False
+
+ # Verify and perform action only if a PkgBotAdmin clicked the button
+ if user_that_clicked: # and full_admin:
+
+ await SlackBot.reaction(
+ action = "add",
+ emoji = "gear",
+ ts = message_ts
+ )
+
+ if button_text == "Approve":
+
+ if button_value_type == "Package":
+ log.info(f"PkgBotAdmin `{username}` is promoting package id: {button_value}")
+
+ await api.package.update(button_value,
+ { "response_url": response_url, "status_updated_by": username })
+ await api.package.promote_package(button_value)
+
+ elif button_value_type == "Trust":
+ log.info(
+ f"PkgBotAdmin `{username}` has approved updates for trust id: {button_value}")
+
+ updates = {
+ "response_url": response_url,
+ "status_updated_by": username,
+ "slack_ts": message_ts
+ }
+
+ trust_object = await models.TrustUpdates.filter(id=button_value).first()
+ await models.TrustUpdates.update_or_create(updates, id=trust_object.id)
+ await api.recipe.recipe_trust_update(trust_object)
+
+ elif button_text == "Deny":
+
+ if button_value_type == "Package":
+ log.info(f"PkgBotAdmin `{username}` has denied package id: {button_value}")
+
+ await api.package.update(button_value,
+ { "response_url": response_url,
+ "status_updated_by": username,
+ "status": "Denied",
+ "notes": "This package was not approved for use in production." }
+ )
+ await api.package.deny_package(button_value)
+
+ if button_value_type == "Trust":
+ log.info(
+ f"PkgBotAdmin `{username}` has denied updates for trust id: {button_value}")
+
+ updates = {
+ "response_url": response_url,
+ "status_updated_by": username,
+ "status": "Denied"
+ }
+
+ trust_object = await models.TrustUpdates.filter(id=button_value).first()
+ await models.TrustUpdates.update_or_create(updates, id=trust_object.id)
+ await api.recipe.recipe_trust_deny(button_value)
+
+ elif button_text == "Acknowledge":
+
+ if button_value_type == "Error":
+ log.info(f"PkgBotAdmin `{username}` has acknowledged error id: {button_value}")
+ return await SlackBot.delete_message(str(message_ts))
+
+ else:
+ log.warning(f"Unauthorized user: `{username}` [{user_id}].")
+
+ blocks = await api.build_msg.unauthorized_msg(username)
+ await SlackBot.post_ephemeral_message(
+ user_id, blocks, channel=channel, text="WARNING: Unauthorized access attempted")
+
+ return { "result": 200 }
+
+ else:
+ log.warning("PkgBot received an invalid request!")
+ return { "result": 500 }
diff --git a/pkgbot/api/slack/build_msg.py b/pkgbot/api/slack/build_msg.py
new file mode 100644
index 0000000..ae79c53
--- /dev/null
+++ b/pkgbot/api/slack/build_msg.py
@@ -0,0 +1,152 @@
+import json
+
+from fastapi import APIRouter, Depends
+
+from pkgbot import api, settings
+from pkgbot.db import models
+from pkgbot.utilities import common as utility
+
+
+log = utility.log
+SlackBot = None
+router = APIRouter(
+ prefix = "/slackbot/build",
+ tags = ["slackbot"],
+ dependencies = [Depends(api.user.verify_admin)],
+ responses = settings.api.custom_responses
+)
+
+async def format_json(the_json, indent=4):
+ return json.dumps(the_json, indent=indent)
+
+
+@router.get("/new-pkg-msg", summary="Build new package message",
+ description="Builds a 'new package' message for Slack after "
+ "a .pkg has been added to the dev environment.")
+async def new_pkg_msg(pkg_object: models.Package_In = Depends(models.Package_In)):
+
+ blocks = [
+ await api.block_builders.brick_header(pkg_object),
+ await api.block_builders.brick_main(pkg_object),
+ await api.block_builders.brick_footer_dev(pkg_object)
+ ]
+
+ for brick in await api.block_builders.brick_button(pkg_object):
+ blocks.append(brick)
+
+ return await format_json(blocks)
+
+
+@router.get("/recipe-error", summary="Build error message",
+ description="Builds an 'error' message for Slack after a recipe has returned an error.")
+async def recipe_error_msg(recipe_id: str, id: int, error: dict):
+
+ redacted_error = await utility.replace_sensitive_strings(error)
+ blocks = await api.block_builders.brick_error(recipe_id, format_json(redacted_error))
+ return format_json(blocks)
+
+
+@router.get("/trust-diff-msg", summary="Build trust diff message",
+ description="Builds a message with the trust diff contents "
+ "for Slack after a recipe's parent trust info has changed.")
+async def trust_diff_msg(id: int, recipe: str, diff_msg: str = None):
+
+ blocks = [
+ await api.block_builders.brick_trust_diff_header(),
+ await api.block_builders.brick_trust_diff_main(recipe)
+ ]
+
+ if diff_msg:
+ blocks.append(await api.block_builders.brick_trust_diff_content(diff_msg))
+
+ blocks.append(await api.block_builders.brick_trust_diff_button(id))
+ return format_json(blocks)
+
+
+@router.get("/deny-pkg-msg", summary="Build deny package message",
+ description="Builds a 'package denied message' for Slack when "
+ "a .pkg is not approved for the production environment.")
+async def deny_pkg_msg(pkg_object: models.Package_In = Depends(models.Package_In)):
+
+ brick_footer = await api.block_builders.brick_footer_dev(pkg_object)
+ brick_footer.get("elements").append(
+ await api.block_builders.brick_footer_denied(pkg_object)
+ )
+
+ blocks = [
+ await api.block_builders.brick_deny_pkg(pkg_object),
+ await api.block_builders.brick_main(pkg_object),
+ brick_footer
+ ]
+
+ return format_json(blocks)
+
+
+@router.get("/deny-trust-msg", summary="Build deny trust message",
+ description="Builds an message for Slack stating a recipe's "
+ "parent trust info changes were not approved.")
+async def deny_trust_msg(
+ trust_object: models.TrustUpdate_In = Depends(models.TrustUpdate_In)):
+
+ blocks = [
+ await api.block_builders.brick_deny_trust(trust_object),
+ await api.block_builders.brick_footer_denied_trust(trust_object)
+ ]
+
+ return format_json(blocks)
+
+
+@router.get("/promote-msg", summary="Build promoted package message",
+ description="Builds a 'package has been promoted' message for Slack "
+ "after a .pkg has been approved for the production environment.")
+async def promote_msg(pkg_object: models.Package_In = Depends(models.Package_In)):
+
+ brick_footer = await api.block_builders.brick_footer_dev(pkg_object)
+ brick_footer.get("elements").append(
+ await api.block_builders.brick_footer_promote(pkg_object)
+ )
+
+ blocks = [
+ await api.block_builders.brick_main(pkg_object),
+ brick_footer
+ ]
+
+ return format_json(blocks)
+
+
+@router.get("/update-trust-success-msg", summary="Build trust update success message",
+ description="Builds a 'success' message for Slack when a "
+ "recipe's trust info is updated successfully.")
+async def update_trust_success_msg(
+ trust_object: models.TrustUpdate_In = Depends(models.TrustUpdate_In)):
+
+ blocks = [
+ await api.block_builders.brick_update_trust_success_msg(trust_object),
+ await api.block_builders.brick_footer_update_trust_success_msg(trust_object)
+ ]
+
+ return format_json(blocks)
+
+
+@router.get("/update-trust-error-msg", summary="Build trust update error message",
+ description="Builds an 'error' message for Slack when a recipe's trust info fails to update.")
+async def update_trust_error_msg(msg: str,
+ trust_object: models.TrustUpdate_In = Depends(models.TrustUpdate_In)):
+
+ return format_json([await api.block_builders.brick_update_trust_error_msg(trust_object, msg)])
+
+
+@router.get("/unauthorized-msg", summary="Build unauthorized message",
+ description="Builds a 'unauthorized' message for Slack when a user attempts to "
+ "perform a Slack interaction with PkgBot that they're not authorized to perform.")
+async def unauthorized_msg(user):
+
+ return format_json(await api.block_builders.unauthorized(user))
+
+
+@router.get("/missing-recipe-msg", summary="Build unauthorized message",
+ description="Builds a 'missing recipe' message for Slack when unable to locate "
+ "a recipe for a requested action.")
+async def missing_recipe_msg(recipe_id, text):
+
+ return format_json(await api.block_builders.missing_recipe_msg(recipe_id, text))
diff --git a/pkgbot/api/slack/send_msg.py b/pkgbot/api/slack/send_msg.py
new file mode 100644
index 0000000..ef9a3d8
--- /dev/null
+++ b/pkgbot/api/slack/send_msg.py
@@ -0,0 +1,185 @@
+import json
+
+from fastapi import APIRouter, Depends
+
+from pkgbot import api, settings
+from pkgbot.db import models
+from pkgbot.utilities import common as utility
+
+
+log = utility.log
+SlackBot = None
+router = APIRouter(
+ prefix = "/slackbot/send",
+ tags = ["slackbot"],
+ dependencies = [Depends(api.user.verify_admin)],
+ responses = settings.api.custom_responses
+)
+
+max_content_size = 1500
+
+
+@router.post("/dev-msg", summary="Send new package message",
+ description="Sends a 'new package' message to Slack after "
+ "a .pkg has been added to the dev environment.")
+async def new_pkg_msg(pkg_object: models.Package_In = Depends(models.Package_In)):
+
+ return await api.bot.SlackBot.post_message(
+ await api.build_msg.new_pkg_msg(pkg_object),
+ text=f"Update for {pkg_object.name}"
+ )
+
+
+@router.post("/promote-msg", summary="Send promoted package message",
+ description="Sends a 'package has been promoted' message to Slack "
+ "after a .pkg has been approved for the production environment.")
+async def promote_msg(pkg_object: models.Package_In = Depends(models.Package_In)):
+
+ blocks = await api.build_msg.promote_msg(pkg_object)
+ text = f"{pkg_object.pkg_name} was promoted to production"
+
+ result = await api.bot.SlackBot.update_message_with_response_url(
+ pkg_object.response_url,
+ blocks,
+ text=text
+ )
+
+ # If the first method fails, try the alternate
+ if json.loads(result.body).get("error") == "expired_url":
+ await api.bot.SlackBot.update_message(
+ blocks,
+ pkg_object.slack_ts,
+ text=text
+ )
+
+ return await api.bot.SlackBot.reaction(
+ action = "remove",
+ emoji = "gear",
+ ts = pkg_object.slack_ts
+ )
+
+
+@router.post("/recipe-error-msg", summary="Send error message",
+ description="Sends an 'error' message to Slack after a recipe has returned an error.")
+async def recipe_error_msg(recipe_id: str, id: int, error: str):
+
+ blocks = await api.build_msg.recipe_error_msg(recipe_id, id, error)
+ return await api.bot.SlackBot.post_message(blocks, text=f"Encountered error in {recipe_id}")
+
+
+@router.post("/trust-diff-msg", summary="Send trust diff message",
+ description="Sends a message with the trust diff contents to "
+ "Slack after a recipe's parent trust info has changed.")
+async def trust_diff_msg(
+ diff_msg: str, trust_object: models.TrustUpdate_In = Depends(models.TrustUpdate_In)):
+
+ if len(diff_msg) > max_content_size:
+ blocks = await api.build_msg.trust_diff_msg(trust_object.id, trust_object.recipe_id)
+ else:
+ blocks = await api.build_msg.trust_diff_msg(
+ trust_object.id, trust_object.recipe_id, diff_msg)
+
+ response = await api.bot.SlackBot.post_message(
+ blocks,
+ text=f"Trust verification failed for `{trust_object.recipe_id}`"
+ )
+
+ trust_object.slack_ts = response.get('ts')
+ await trust_object.save()
+
+ if len(diff_msg) > max_content_size:
+ response = await api.bot.SlackBot.file_upload(
+ content = diff_msg,
+ filename = f"{trust_object.recipe_id}.diff",
+ filetype = "diff",
+ title = trust_object.recipe_id,
+ text = f"Diff Output for {trust_object.recipe_id}",
+ thread_ts = trust_object.slack_ts
+ )
+
+ return response
+
+
+@router.put("/update-trust-success-msg", summary="Send trust update success message",
+ description="Sends a 'success' message to Slack when "
+ "a recipe's trust info is updated successfully.")
+async def update_trust_success_msg(
+ trust_object: models.TrustUpdate_In = Depends(models.TrustUpdate_In)):
+
+ blocks = await api.build_msg.update_trust_success_msg(trust_object)
+
+ response = await api.bot.SlackBot.update_message_with_response_url(
+ trust_object.dict().get("response_url"),
+ blocks,
+ text=f"Successfully updated trust info for {trust_object.recipe_id}"
+ )
+
+ if response.status_code == 200:
+ await api.bot.SlackBot.reaction(
+ action = "remove",
+ emoji = "gear",
+ ts = trust_object.slack_ts
+ )
+
+ return response
+
+
+@router.put("/update-trust-error-msg", summary="Send trust update error message",
+ description="Sends an 'error' message to Slack when a recipe's trust info fails to update.")
+async def update_trust_error_msg(msg: str,
+ trust_object: models.TrustUpdate_In = Depends(models.TrustUpdate_In)):
+
+ blocks = await api.build_msg.update_trust_error_msg(msg, trust_object)
+
+ return await api.bot.SlackBot.update_message_with_response_url(
+ trust_object.dict().get("response_url"),
+ blocks,
+ text=f"Failed to update trust info for {trust_object.recipe_id}"
+ )
+
+
+@router.put("/deny-pkg-msg", summary="Send deny package message",
+ description="Sends a 'package denied message' to Slack when "
+ "a .pkg is not approved for the production environment.")
+async def deny_pkg_msg(pkg_object: models.Package_In = Depends(models.Package_In)):
+
+ blocks = await api.build_msg.deny_pkg_msg(pkg_object)
+
+ response = await api.bot.SlackBot.update_message_with_response_url(
+ pkg_object.dict().get("response_url"),
+ blocks,
+ text=f"{pkg_object.pkg_name} was not approved for production"
+ )
+
+ if response.status_code == 200:
+ await api.bot.SlackBot.reaction(
+ action = "remove",
+ emoji = "gear",
+ ts = pkg_object.slack_ts
+ )
+
+ return response
+
+
+@router.put("/deny-trust-msg", summary="Send deny trust message",
+ description="Send an message to Slack stating a recipe's "
+ "parent trust info changes were not approved.")
+async def deny_trust_msg(
+ trust_object: models.TrustUpdate_In = Depends(models.TrustUpdate_In)):
+
+ blocks = await api.build_msg.deny_trust_msg(trust_object)
+
+ response = await api.bot.SlackBot.update_message_with_response_url(
+ trust_object.dict().get("response_url"),
+ blocks,
+ text=f"Trust info for {trust_object.recipe_id} was not approved"
+ )
+
+ if response.status_code == 200:
+ await api.bot.SlackBot.reaction(
+ action = "remove",
+ emoji = "gear",
+ ts = trust_object.slack_ts
+ )
+
+ return response
diff --git a/api/user.py b/pkgbot/api/user.py
similarity index 78%
rename from api/user.py
rename to pkgbot/api/user.py
index 349b57d..581b105 100644
--- a/api/user.py
+++ b/pkgbot/api/user.py
@@ -1,19 +1,17 @@
-#!/usr/local/autopkg/python
-
from fastapi import APIRouter, Depends, HTTPException, Response, status
from fastapi.security import OAuth2PasswordBearer
-import utils
-from db import models
-from api import settings
+from pkgbot import settings
+from pkgbot.db import models
+from pkgbot.utilities import common as utility
-log = utils.log
+log = utility.log
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth/token")
router = APIRouter(
prefix = "/user",
tags = ["user"],
- responses = settings.custom_responses
+ responses = settings.api.custom_responses
)
@@ -44,18 +42,18 @@ async def get_current_user(token: str = Depends(oauth2_scheme)):
)
-async def verify_admin(response: Response,
+async def verify_admin(response: Response,
user: models.PkgBotAdmin_In = Depends(get_current_user)):
if not user.full_admin:
log.debug("User is NOT an admin")
- raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,
+ raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,
detail="You are not authorized to utilize this endpoint.")
# log.debug("User is an admin")
-@router.get("s/", summary="Get all users", description="Get all users in the database.",
+@router.get("s/", summary="Get all users", description="Get all users in the database.",
dependencies=[Depends(verify_admin)])
async def get_users():
@@ -64,30 +62,30 @@ async def get_users():
return { "total": len(users), "users": users }
-@router.post("/create", summary="Create a user", description="Creates a new PkgBot user.",
- dependencies=[Depends(verify_admin)], response_model=models.PkgBotAdmin_Out,
+@router.post("/create", summary="Create a user", description="Creates a new PkgBot user.",
+ dependencies=[Depends(verify_admin)], response_model=models.PkgBotAdmin_Out,
response_model_exclude={ "slack_id", "jps_token" }, response_model_exclude_unset=True)
-async def create_user(response: Response,
+async def create_user(response: Response,
user: models.PkgBotAdmin_In = Depends(models.PkgBotAdmin_In)):
if await get_user(user):
- raise HTTPException(status_code=status.HTTP_409_CONFLICT,
- detail="The user `{}` already exists.".format(user.username))
+ raise HTTPException(status_code=status.HTTP_409_CONFLICT,
+ detail=f"The user `{user.username}` already exists.")
return await create_or_update_user(user)
-@router.put("/update", summary="Update a user", description="Updates an existing PkgBot user.",
- dependencies=[Depends(verify_admin)], response_model=models.PkgBotAdmin_Out,
+@router.put("/update", summary="Update a user", description="Updates an existing PkgBot user.",
+ dependencies=[Depends(verify_admin)], response_model=models.PkgBotAdmin_Out,
response_model_exclude={ "slack_id", "jps_token" }, response_model_exclude_unset=True)
-async def update_user(response: Response,
+async def update_user(response: Response,
user: models.PkgBotAdmin_In = Depends(models.PkgBotAdmin_In)):
return await create_or_update_user(user)
-@router.get("/whoami", summary="Get user's info",
- description="Get the currently authenticated users information.",
+@router.get("/whoami", summary="Get user's info",
+ description="Get the currently authenticated users information.",
dependencies=[Depends(get_current_user)], response_model=models.PkgBotAdmin_Out)
async def whoami(user: models.PkgBotAdmin_In = Depends(get_current_user)):
diff --git a/pkgbot/api/views.py b/pkgbot/api/views.py
new file mode 100644
index 0000000..a786b19
--- /dev/null
+++ b/pkgbot/api/views.py
@@ -0,0 +1,122 @@
+import os
+import shutil
+
+from datetime import datetime
+
+from fastapi import APIRouter, Depends, Request, UploadFile
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+
+from pkgbot import config
+from pkgbot.utilities import common as utility
+from pkgbot.api import auth, package, recipe
+
+
+log = utility.log
+config = config.load_config()
+
+
+def template_filter_datetime(date, date_format="%Y-%m-%d %I:%M:%S"):
+
+ if date:
+ converted = datetime.fromisoformat(str(date))
+ return converted.strftime(date_format)
+
+
+session = { "logged_in": False }
+templates = Jinja2Templates(directory=config.PkgBot.get("jinja_templates"))
+templates.env.filters["strftime"] = template_filter_datetime
+router = APIRouter(
+ tags = ["view"],
+ include_in_schema = False
+)
+
+
+@router.get("/", response_class=HTMLResponse)
+async def index(request: Request):
+
+ if request.state.user:
+ session["logged_in"] = True
+ else:
+ session["logged_in"] = False
+
+ return templates.TemplateResponse("index.html", { "request": request, "session": session })
+
+
+# @router.get("/login", response_class=HTMLResponse)
+# async def userlogin(request: Request):
+
+# return templates.TemplateResponse("login.html", { "request": request, "session": session })
+
+
+@router.get("/packages", response_class=HTMLResponse)
+async def package_history(request: Request, user = Depends(auth.login_manager)):
+
+ session["logged_in"] = True
+ pkgs = await package.get_packages()
+
+ table_headers = [
+ "", "", "Name", "Version", "Status", "Updated By",
+ "Packaged", "Promoted", "COMMON", "Flags", "Notes"
+ ]
+
+ return templates.TemplateResponse("packages.html",
+ { "request": request, "session": session,
+ "table_headers": table_headers, "packages": pkgs.get("packages") })
+
+
+@router.get("/package/{id}", response_class=HTMLResponse)
+async def get_package(request: Request, user = Depends(auth.login_manager)):
+
+ session["logged_in"] = True
+ pkg = await package.get_package_by_id(request.path_params['id'])
+
+ return templates.TemplateResponse("package.html",
+ { "request": request, "session": session, "package": pkg })
+
+
+@router.get("/edit/{id}", response_class=HTMLResponse)
+async def edit(request: Request, user = Depends(auth.login_manager)):
+
+ pkg = await package.get_package_by_id(request.path_params['id'])
+
+ return templates.TemplateResponse("edit.html",
+ { "request": request, "session": session, "package": pkg })
+
+
+@router.get("/recipes", response_class=HTMLResponse)
+async def recipe_list(request: Request, user = Depends(auth.login_manager)):
+
+ session["logged_in"] = True
+ pkgs = await recipe.get_recipes()
+
+ table_headers = [ "ID", "Recipe ID", "Name", "Enable", "Manual Only",
+ "Pkg Only", "Last Ran", "Schedule", "Notes" ]
+
+ return templates.TemplateResponse("recipes.html",
+ { "request": request, "session": session,
+ "table_headers": table_headers, "recipes": pkgs.get("recipes") })
+
+
+@router.get("/recipe/{id}", response_class=HTMLResponse)
+async def get_recipe(request: Request, user = Depends(auth.login_manager)):
+
+ session["logged_in"] = True
+ pkg = await recipe.get_by_id(request.path_params['id'])
+
+ return templates.TemplateResponse("recipe.html",
+ { "request": request, "session": session, "recipe": pkg })
+
+
+@router.post("/icons")
+async def upload_icon(icon: UploadFile, user = Depends(auth.login_manager)):
+
+ pkg_dir = os.path.abspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir))
+
+ try:
+ with open(f"{pkg_dir}/static/icons/{icon.filename}", "wb") as icon_obj:
+ shutil.copyfileobj(icon.file, icon_obj)
+ finally:
+ await icon.close()
+
+ return { "results": 200, "icon": icon.filename }
diff --git a/pkgbot/config.py b/pkgbot/config.py
new file mode 100644
index 0000000..113a544
--- /dev/null
+++ b/pkgbot/config.py
@@ -0,0 +1,77 @@
+import argparse
+import os
+import yaml
+
+from functools import lru_cache
+
+from pydantic import BaseSettings
+from pydantic.env_settings import SettingsSourceCallable
+
+
+def yml_config_setting(settings: BaseSettings):
+
+ with open(settings.__config__.config_file) as file:
+ return yaml.safe_load(file)
+
+
+def instantiate_config():
+
+ class PkgBot_Configuration(BaseSettings):
+
+ class Config:
+
+ arbitrary_types_allowed = True
+ config_file = os.environ.get("PKGBOT_CONFIG")
+ env_prefix = "PKGBOT_"
+ # allow extra options so we can detect legacy configuration files
+ extra = "allow"
+
+ @classmethod
+ def customise_sources(
+ cls,
+ init_settings: SettingsSourceCallable,
+ env_settings: SettingsSourceCallable,
+ file_secret_settings: SettingsSourceCallable,
+ ):
+ # Add load from yml file, change priority and remove file secret option
+ return init_settings, yml_config_setting, env_settings
+
+ return PkgBot_Configuration
+
+
+@lru_cache()
+def load_config(cli_args=None):
+
+ # print(f'PkgBot.Load_Config:\n\tAll calling args: {cli_args}')
+ parser = argparse.ArgumentParser(description="PkgBot Main.")
+ parser.add_argument(
+ '--pkgbot_config', '-pc', metavar='./pkgbot.config', default=None,
+ type=str, required=False, help='A defined pkgbot configuration file.'
+ )
+ args, _ = parser.parse_known_args(cli_args)
+ # print(f'PkgBot.Load_Config:\n\tArgparse args: {args}')
+
+ pkg_dir = os.path.abspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir))
+
+ config_file = (
+ args.pkgbot_config or
+ os.environ.get(
+ "PKGBOT_CONFIG",
+ os.path.join(pkg_dir, "Settings/pkgbot_config.yaml")
+ )
+ )
+
+ if not os.path.exists(config_file):
+ raise("The specified config file does not exist.")
+
+ os.environ["PKGBOT_CONFIG"] = config_file
+ PkgBot_Configuration = instantiate_config()
+ config = PkgBot_Configuration()
+
+ if config.AutoPkg.get('binary') is None:
+ config.AutoPkg["binary"] = "/usr/local/bin/autopkg"
+
+ if config.Git.get('binary') is None:
+ config.Git["binary"] = "/usr/bin/git"
+
+ return config
diff --git a/execute/__init__.py b/pkgbot/db/__init__.py
similarity index 100%
rename from execute/__init__.py
rename to pkgbot/db/__init__.py
diff --git a/db/models.py b/pkgbot/db/models.py
similarity index 58%
rename from db/models.py
rename to pkgbot/db/models.py
index 7b04167..75b6422 100644
--- a/db/models.py
+++ b/pkgbot/db/models.py
@@ -1,5 +1,4 @@
-#!/usr/local/autopkg/python
-
+from pydantic import BaseModel
from tortoise import fields
from tortoise.models import Model
from tortoise.contrib.pydantic import pydantic_model_creator
@@ -11,10 +10,11 @@ class Packages(Model):
name = fields.CharField(64)
version = fields.CharField(64)
pkg_name = fields.CharField(256, null=True)
- jps_url = fields.CharField(64, null=True)
- icon_id = fields.CharField(64, null=True)
- jps_id_dev = fields.IntField(unique=True, null=True)
- jps_id_prod = fields.IntField(unique=True, null=True)
+ jps_url = fields.CharField(64, null=True) # DEPRECATED | TO BE REMOVED
+ icon_id = fields.CharField(64, null=True) # DEPRECATED | TO BE REMOVED
+ icon = fields.CharField(1024, null=True)
+ jps_id_dev = fields.IntField(unique=True, null=True) # DEPRECATED | TO BE REMOVED
+ jps_id_prod = fields.IntField(unique=True, null=True) # DEPRECATED | TO BE REMOVED
packaged_date = fields.DatetimeField(auto_now_add=True)
promoted_date = fields.DatetimeField(null=True, default=None)
last_update = fields.DatetimeField(auto_now=True)
@@ -36,13 +36,26 @@ class Recipes(Model):
recipe_id = fields.CharField(512, unique=True)
name = fields.CharField(64)
enabled = fields.BooleanField(default=True)
+ manual_only = fields.BooleanField(default=False)
pkg_only = fields.BooleanField(default=False)
last_ran = fields.DatetimeField(null=True, default=None)
+ recurring_fail_count = fields.IntField(null=True, default=0)
schedule = fields.IntField(default=0)
notes = fields.CharField(4096, null=True)
Recipe_Out = pydantic_model_creator(Recipes, name="Recipe_Out")
Recipe_In = pydantic_model_creator(Recipes, name="Recipe_In", exclude_readonly=True)
+# Recipe_Filter = pydantic_model_creator(
+# Recipes, name="Recipe_Filter", exclude_readonly=True,
+# exclude=('id', "recipe_id", "name", "last_ran", "notes"),
+# optional=( "enabled", "manual_only", "pkg_only", "recurring_fail_count", "schedule"))
+
+class Recipe_Filter(BaseModel):
+ enabled: bool | None = None
+ manual_only: bool | None = None
+ pkg_only: bool | None = None
+ recurring_fail_count: int | None = None
+ schedule: int | None = None
class PkgBotAdmins(Model):
@@ -72,3 +85,37 @@ class ErrorMessages(Model):
ErrorMessage_Out = pydantic_model_creator(ErrorMessages, name="ErrorMessage_Out")
ErrorMessage_In = pydantic_model_creator(
ErrorMessages, name="ErrorMessage_In", exclude_readonly=True)
+
+
+class TrustUpdates(Model):
+ id = fields.IntField(pk=True)
+ recipe_id = fields.CharField(1024)
+ slack_ts = fields.CharField(32, null=True)
+ slack_channel = fields.CharField(32, null=True)
+ response_url = fields.CharField(1024, null=True)
+ status_updated_by = fields.CharField(64, default="PkgBot")
+ last_update = fields.DatetimeField(auto_now=True)
+ status = fields.CharField(64, null=True)
+
+TrustUpdate_Out = pydantic_model_creator(TrustUpdates, name="TrustUpdate_Out")
+TrustUpdate_In = pydantic_model_creator(
+ TrustUpdates, name="TrustUpdate_In", exclude_readonly=True)
+
+
+##### May make this a Tortoise Model, to support tracking who/what generated each command
+class AutoPkgCMD(BaseModel):
+ ignore_parent_trust: bool = False
+ match_pkg: str | None = None
+ # overrides: str | None = None # Not yet supported
+ pkg_only: bool = False
+ promote: bool = False
+ verbose: str = "vvv"
+
+
+class AutoPkgTaskResults(BaseModel):
+ event: str
+ event_id: str = ""
+ recipe_id: str
+ success: str
+ stdout: str
+ stderr: str
diff --git a/pkgbot/settings/__init__.py b/pkgbot/settings/__init__.py
new file mode 100644
index 0000000..6ed0a52
--- /dev/null
+++ b/pkgbot/settings/__init__.py
@@ -0,0 +1,3 @@
+from . import api
+from . import celery
+from . import db
diff --git a/api/settings.py b/pkgbot/settings/api.py
similarity index 69%
rename from api/settings.py
rename to pkgbot/settings/api.py
index d136146..3980044 100644
--- a/api/settings.py
+++ b/pkgbot/settings/api.py
@@ -1,6 +1,4 @@
-#!/usr/local/autopkg/python
-
-from fastapi import status
+from fastapi import status
from tortoise.contrib.fastapi import HTTPNotFoundError
@@ -31,27 +29,13 @@
}
]
-TORTOISE_CONFIG = {
- "connections": {
- # "default": "sqlite://:memory:"
- "default": "sqlite://db/db.sqlite3"
- },
- "apps": {
- "app": {
- "models": [ "db.models" ],
- "default_connection": "default"
- }
- },
- "use_tz": False,
- "timezone": "America/Phoenix"
-}
custom_responses = {
# 404: {"description": "Item not found"},
# 302: {"description": "The item was moved"},
- status.HTTP_401_UNAUTHORIZED: { "description":
+ status.HTTP_401_UNAUTHORIZED: { "description":
"You must authenticate before utilizing this endpoint." },
- status.HTTP_403_FORBIDDEN: { "description":
+ status.HTTP_403_FORBIDDEN: { "description":
"You are not authorized to utilize this endpoint." },
status.HTTP_404_NOT_FOUND: { "model": HTTPNotFoundError },
status.HTTP_409_CONFLICT: { "description": "The object already exists." }
diff --git a/pkgbot/settings/celery.py b/pkgbot/settings/celery.py
new file mode 100644
index 0000000..6915239
--- /dev/null
+++ b/pkgbot/settings/celery.py
@@ -0,0 +1,48 @@
+import os
+
+from functools import lru_cache
+from kombu import Queue
+
+from pkgbot import config
+
+
+config = config.load_config()
+
+
+def route_task(name, args, kwargs, options, task=None, **kw):
+
+ if ":" in name:
+ queue, _ = name.split(":")
+ return {"queue": queue}
+
+ return {"queue": "celery"}
+
+
+class CeleryConfig:
+ broker_url: str = os.environ.get("broker_url", config.Celery.get("broker_url"))
+ # result_backend: str = os.environ.get("result_backend", "rpc://")
+ result_backend: str = os.environ.get("result_backend", f"db+sqlite:///{config.Database.get('location')}")
+
+ task_queues: list = (
+ Queue("pkgbot", queue_arguments={"x-max-priority": 10}),
+ Queue("autopkg", queue_arguments={"x-max-priority": 10}),
+ Queue("git", queue_arguments={"x-max-priority": 10}),
+ )
+
+ task_routes = (route_task,)
+ task_default_priority = 5
+ task_queue_max_priority = 10
+ task_acks_late = True
+ worker_prefetch_multiplier = 1
+
+
+@lru_cache()
+def get_settings():
+
+ config_cls_dict = { "pkgbot": CeleryConfig, }
+ config_name = os.environ.get("CELERY_CONFIG", "pkgbot")
+ config_cls = config_cls_dict[config_name]
+ return config_cls()
+
+
+settings = get_settings()
diff --git a/pkgbot/settings/db.py b/pkgbot/settings/db.py
new file mode 100644
index 0000000..29d61b9
--- /dev/null
+++ b/pkgbot/settings/db.py
@@ -0,0 +1,19 @@
+from pkgbot import config
+
+
+config = config.load_config()
+
+TORTOISE_CONFIG = {
+ "connections": {
+ # "default": "sqlite://:memory:"
+ "default": f"sqlite:{config.Database.get('location')}"
+ },
+ "apps": {
+ "app": {
+ "models": [ "pkgbot.db.models" ],
+ "default_connection": "default"
+ }
+ },
+ "use_tz": False,
+ "timezone": "America/Phoenix"
+}
diff --git a/pkgbot/static/icons/denied.png b/pkgbot/static/icons/denied.png
new file mode 100644
index 0000000..909e984
Binary files /dev/null and b/pkgbot/static/icons/denied.png differ
diff --git a/pkgbot/static/icons/error.png b/pkgbot/static/icons/error.png
new file mode 100644
index 0000000..15aaf4a
Binary files /dev/null and b/pkgbot/static/icons/error.png differ
diff --git a/pkgbot/static/icons/metapackage.png b/pkgbot/static/icons/metapackage.png
new file mode 100644
index 0000000..dd66cad
Binary files /dev/null and b/pkgbot/static/icons/metapackage.png differ
diff --git a/pkgbot/static/icons/package.png b/pkgbot/static/icons/package.png
new file mode 100644
index 0000000..b8fb9f7
Binary files /dev/null and b/pkgbot/static/icons/package.png differ
diff --git a/pkgbot/static/icons/permission_denied.png b/pkgbot/static/icons/permission_denied.png
new file mode 100644
index 0000000..9858a58
Binary files /dev/null and b/pkgbot/static/icons/permission_denied.png differ
diff --git a/pkgbot/static/icons/warning.png b/pkgbot/static/icons/warning.png
new file mode 100644
index 0000000..bc26f6b
Binary files /dev/null and b/pkgbot/static/icons/warning.png differ
diff --git a/pkgbot/tasks/__init__.py b/pkgbot/tasks/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pkgbot/tasks/task.py b/pkgbot/tasks/task.py
new file mode 100644
index 0000000..b1009a9
--- /dev/null
+++ b/pkgbot/tasks/task.py
@@ -0,0 +1,490 @@
+import asyncio
+import hashlib
+import json
+import os
+import requests
+
+from datetime import datetime
+
+import git
+from celery import Celery, group, chain
+
+from pkgbot import config, settings
+from pkgbot.db import models
+from pkgbot.tasks import task_utils
+from pkgbot.utilities import common as utility
+
+
+config = config.load_config()
+log = utility.log
+celery = Celery()
+celery.config_from_object(settings.celery.settings)
+
+
+@celery.task(name="pkgbot:send_webhook", namebind=True)
+def send_webhook(task_id):
+ """ Sends webhook after a task is complete. """
+
+ pkgbot_server, headers = task_utils.api_url_helper()
+ data = { "task_id": task_id }
+
+ headers["x-pkgbot-signature"] = asyncio.run(utility.compute_hex_digest(
+ config.PkgBot.get("webhook_secret").encode("UTF-8"),
+ str(data).encode("UTF-8"),
+ hashlib.sha512
+ ))
+
+ requests.post(f"{pkgbot_server}/autopkg/receive",
+ headers=headers,
+ data=json.dumps(data),
+ )
+
+
+@celery.task(name="git:pull_private_repo", bind=True)
+def git_pull_private_repo(self):
+ """Perform a `git pull` for the local private repo"""
+
+ log.info("Checking for private repo updates...")
+
+ repo_primary_branch = config.Git.get("repo_primary_branch")
+ repo_push_branch = config.Git.get("repo_push_branch")
+ stashed = False
+ use_remote_push = False
+
+ try:
+
+ private_repo = git.Repo(os.path.expanduser(config.Git.get("local_repo_dir")))
+
+ if private_repo.is_dirty():
+ _ = private_repo.git.stash()
+ stashed = True
+
+ active_branch = private_repo.active_branch
+ local_branches = [ branch.name for branch in private_repo.branches ]
+
+ _ = private_repo.remotes.origin.fetch()
+ # remote_branches = [ ref.name.split("/")[1] for ref in private_repo.remote().refs ]
+
+ if active_branch != repo_primary_branch:
+ _ = private_repo.git.checkout(repo_primary_branch)
+
+ if repo_push_branch in local_branches:
+ push_branch_commits_ahead, push_branch_commits_behind = task_utils.compare_branch_heads(
+ private_repo, repo_push_branch, repo_primary_branch)
+
+ if push_branch_commits_ahead == 0 and push_branch_commits_behind > 0:
+ # private_repo.delete_head(repo_push_branch)
+ # For safety, just renaming the branch for now and after a bit of real world
+ # testing, switch to deleting the branch
+ timestamp = asyncio.run(
+ utility.datetime_to_string(str(datetime.now()), "%Y-%m-%d_%I-%M-%S"))
+ private_repo.branches[repo_push_branch].rename(
+ f"{repo_push_branch}_{timestamp}")
+
+ elif push_branch_commits_ahead > 0:
+ use_remote_push = True
+
+ primary_branch_commits_ahead, primary_branch_commits_behind = task_utils.compare_branch_heads(
+ private_repo, repo_primary_branch, repo_primary_branch)
+
+ if primary_branch_commits_ahead != 0 and repo_primary_branch != repo_push_branch:
+ log.error("Local primary branch is ahead of remote primary branch.")
+ raise
+
+ _ = private_repo.remotes.origin.pull()
+
+ results_git_pull_command = {
+ "stdout": "Success",
+ "stderr": "",
+ "status": 0,
+ "success": True
+ }
+
+ log.info("Successfully updated private repo")
+
+ except Exception as error:
+
+ results_git_pull_command = {
+ "stdout": "Error occurred during git operation",
+ "stderr": error,
+ "status": 1,
+ "success": False
+ }
+
+ finally:
+
+ if use_remote_push:
+ _ = private_repo.git.checkout(repo_push_branch)
+
+ if stashed:
+ private_repo.git.stash("pop")
+
+##### This if statement can be removed after further real world testing...
+ if not results_git_pull_command["success"]:
+ log.error(f"stdout:\n{results_git_pull_command['stdout']}")
+ log.error(f"stderr:\n{results_git_pull_command['stderr']}")
+
+ results_git_pull_command |= {
+ "event": "private_git_pull",
+ "task_id": self.request.id
+ }
+
+ return results_git_pull_command
+
+
+@celery.task(name="autopkg:repo_update", bind=True)
+def autopkg_repo_update(self):
+ """Performs an `autopkg repo-update all`"""
+
+ log.info("Updating parent recipe repos...")
+ autopkg_repo_update_command = f"{config.AutoPkg.get('binary')} repo-update all --prefs=\'{os.path.abspath(config.JamfPro_Dev.get('autopkg_prefs'))}\'"
+
+ if task_utils.get_user_context():
+ autopkg_repo_update_command = f"su - {task_utils.get_console_user()} -c \"{autopkg_repo_update_command}\""
+
+ results_autopkg_repo_update = utility.execute_process(autopkg_repo_update_command)
+
+##### This if statement can be removed after further real world testing...
+ if not results_autopkg_repo_update["success"]:
+ log.error("Failed to update parent recipe repos")
+ log.error(f"stdout:\n{results_autopkg_repo_update['stdout']}")
+ log.error(f"stderr:\n{results_autopkg_repo_update['stderr']}")
+
+ results_autopkg_repo_update |= {
+ "event": "autopkg_repo_update",
+ "task_id": self.request.id
+ }
+
+ return results_autopkg_repo_update
+
+
+@celery.task(name="autopkg:run", bind=True)
+def autopkg_run(self, recipes: list, autopkg_options: models.AutoPkgCMD | dict, called_by: str):
+ """Creates parent and individual recipe tasks.
+
+ Args:
+ recipes (list): A list of recipe dicts, which contain their configurations
+ autopkg_options (models.AutoPkgCMD | dict): AutoPkg CLI options
+ called_by (str): From what/where the task was executed
+
+ Returns:
+ dict: Dict describing the results of the ran process
+ """
+
+ # log.debug(f"Calling autopkg_options: {autopkg_options}")
+ # log.debug(f"recipes: {recipes}")
+
+ # Track all child tasks that are queued by this parent task
+ queued_tasks = []
+ promote = autopkg_options.pop("promote", False)
+
+ if not promote and not autopkg_options.get("ignore_parent_trust"):
+
+ # Run checks if we're not promoting the recipe
+ task_group = group([ autopkg_repo_update.signature(), git_pull_private_repo.signature() ])
+ tasks_results = (task_group.apply_async(
+ queue='autopkg', priority=7)).get(disable_sync_subtasks=False)
+
+ # Check results
+ for task_result in tasks_results:
+ if not task_result["success"]:
+ send_webhook.apply_async((self.request.id,), queue='autopkg', priority=9)
+ queued_tasks.append(task_result["task_id"])
+
+ for recipe in recipes:
+
+ # log.debug(f"recipe: {recipe}")
+ recipe_id = recipe.get("recipe_id")
+
+##### Not yet supported
+ # if autopkg_options["override_keys"]:
+ # for override_key in autopkg_options["override_keys"]:
+ # extra_options = f"{extra_options} --key '{override_key}'"
+##### How will the extra_options be passed?
+
+ if not promote:
+
+ if (
+ called_by == "schedule" and
+ not task_utils.check_recipe_schedule(recipe.get("schedule"), recipe.get("last_ran"))
+ ):
+ log.debut(f"Recipe {recipe_id} is out of schedule")
+ continue
+
+ _ = autopkg_options.pop("match_pkg", None)
+ _ = autopkg_options.pop("pkg_only", None)
+ _ = autopkg_options.pop("pkg_id", None)
+
+ # If ignore parent trust, don't run autopkg_verify_trust
+ if autopkg_options.get("ignore_parent_trust"):
+
+ queued_task = run_recipe.apply_async(
+ ({"success": True}, recipe_id, autopkg_options, called_by),
+ queue='autopkg',
+ priority=4
+ )
+
+ queued_tasks.append(queued_task.id)
+
+ else:
+
+ # Verify trust info and wait
+ chain_results = chain(
+ autopkg_verify_trust.signature(
+ (recipe_id, autopkg_options, called_by),
+ queue='autopkg',
+ priority=2
+ ) |
+ run_recipe.signature(
+ (recipe_id, autopkg_options, called_by),
+ queue='autopkg',
+ priority=3
+ )
+ )()
+
+ queued_tasks.extend([chain_results.parent, chain_results.task_id])
+
+ else:
+ log.info(f"Promoting to production: {autopkg_options['match_pkg']}")
+
+ autopkg_options |= {
+ "ignore_parent_trust": True,
+ "prefs": os.path.abspath(config.JamfPro_Prod.get("autopkg_prefs")),
+ "promote_recipe_id": recipe.get("recipe_id"),
+ "verbose": autopkg_options.get('verbose', 'vv')
+ }
+
+ if recipe.get("pkg_only"):
+ # Only upload the .pkg, do not create/update a Policy
+ recipe_id = config.JamfPro_Prod.get("recipe_template_pkg_only")
+ autopkg_options |= { "pkg_only": True }
+ else:
+ recipe_id = config.JamfPro_Prod.get("recipe_template")
+
+ queued_task = run_recipe.apply_async(
+ ({"event": "promote", "id": autopkg_options.pop("pkg_id")},
+ recipe_id,
+ autopkg_options,
+ called_by),
+ queue='autopkg', priority=4
+ )
+
+ queued_tasks.append(queued_task.id)
+
+ return { "Queued background tasks": queued_tasks }
+
+
+@celery.task(name="autopkg:run_recipe", bind=True)
+def run_recipe(self, parent_task_results: dict, recipe_id: str,
+ autopkg_options: models.AutoPkgCMD | dict, called_by: str):
+ """Runs the passed recipe id against `autopkg run`.
+
+ Args:
+ parent_task_results (dict): Results from the calling task
+ recipe_id (str): Recipe ID of a recipe
+ autopkg_options (models.AutoPkgCMD | dict): AutoPkg CLI options
+ called_by (str): From what/where the task was executed
+
+ Returns:
+ dict: Dict describing the results of the ran process
+ """
+
+ run_type = "recipe_run_prod" if parent_task_results.get("event") == "promote" else "recipe_run_dev"
+
+ # Verify not a promote run and parent tasks results were success
+ if (
+ run_type != "recipe_run_prod"
+ and parent_task_results
+ and not parent_task_results["success"]
+ ):
+
+ # event_id = ""
+
+ if f"{recipe_id}: FAILED" in parent_task_results["stderr"]:
+ log_msg = "Failed to verify trust info for"
+ event_type = "verify_trust_info"
+
+ elif "Didn't find a recipe for" in parent_task_results["stdout"]:
+ log_msg = "Failed to locate"
+ event_type = "error"
+
+ else:
+ # Generic error
+ log_msg = "Unknown failure occurred on"
+ event_type = "error"
+
+ log.error(f"{log_msg} recipe: {recipe_id}")
+ send_webhook.apply_async((self.request.id,), queue='autopkg', priority=9)
+
+ return {
+ "event": event_type,
+ # "event_id": event_id,
+ "called_by": called_by,
+ "recipe_id": recipe_id,
+ "success": parent_task_results["success"],
+ "stdout": parent_task_results["stdout"],
+ "stderr": parent_task_results["stderr"]
+ }
+
+ else:
+ log.info(f"Creating `autopkg run` task for recipe: {recipe_id}")
+
+ # Generate AutoPkg options
+ options = task_utils.generate_autopkg_args(**autopkg_options)
+ # Build the autopkg command
+ cmd = f"{config.AutoPkg.get('binary')} run {recipe_id} {options}"
+
+ if task_utils.get_user_context():
+ cmd = f"su - {task_utils.get_console_user()} -c \"{cmd}\""
+
+ # log.debug(f"Command to execute: {cmd}")
+ results = utility.execute_process(cmd)
+
+ # Send task complete notification
+ send_webhook.apply_async((self.request.id,), queue='autopkg', priority=9)
+
+ return {
+ "event": run_type,
+ "event_id": parent_task_results.get("id"),
+ "recipe_id": recipe_id,
+ "success": results["success"],
+ "stdout": results["stdout"],
+ "stderr": results["stderr"],
+ }
+
+
+@celery.task(name="autopkg:verify-trust", bind=True)
+def autopkg_verify_trust(self, recipe_id: str,
+ autopkg_options: models.AutoPkgCMD | dict, called_by: str):
+ """Runs the passed recipe id against `autopkg verify-trust-info`.
+
+ Args:
+ recipe_id (str): Recipe ID of a recipe
+ autopkg_options (models.AutoPkgCMD | dict): AutoPkg CLI options
+ called_by (str): From what/where the task was executed
+
+ Returns:
+ dict: Dict describing the results of the ran process
+ """
+
+ log.info(f"Verifying trust info for: {recipe_id}")
+
+ # Not overriding verbose when verifying trust info
+ _ = autopkg_options.pop('verbose')
+
+ autopkg_options |= {
+ "prefs": os.path.abspath(config.JamfPro_Dev.get("autopkg_prefs")),
+ "verbose": "vvv"
+ }
+
+ # Generate AutoPkg options
+ options = task_utils.generate_autopkg_args(**autopkg_options)
+ # Build the autopkg command
+ cmd = f"{config.AutoPkg.get('binary')} verify-trust-info {recipe_id} {options}"
+
+ if task_utils.get_user_context():
+ cmd = f"su - {task_utils.get_console_user()} -c \"{cmd}\""
+
+ # log.debug(f"Command to execute: {cmd}")
+ results = utility.execute_process(cmd)
+
+ if called_by in {"api", "slack"} and not self.request.parent_id:
+ send_webhook.apply_async((self.request.id,), queue='autopkg', priority=9)
+
+ return {
+ "event": "verify_trust_info",
+ "called_by": called_by,
+ "recipe_id": recipe_id,
+ "success": results["success"],
+ "stdout": results["stdout"],
+ "stderr": results["stderr"],
+ }
+
+ return results
+
+
+@celery.task(name="autopkg:update-trust", bind=True)
+def autopkg_update_trust(self, recipe_id: str,
+ autopkg_options: models.AutoPkgCMD | dict, trust_id: int = None):
+ """Runs the passed recipe id against `autopkg update-trust-info`.
+
+ Args:
+ recipe_id (str): Recipe ID of a recipe
+ autopkg_options (models.AutoPkgCMD | dict): AutoPkg CLI options
+ trust_id (int): The database id to associate the results to the record
+
+ Returns:
+ dict: Dict describing the results of the ran process
+ """
+
+ log.info(f"Updating trust info for: {recipe_id}")
+
+ # Generate AutoPkg options
+ autopkg_options = task_utils.generate_autopkg_args(
+ prefs=os.path.abspath(config.JamfPro_Dev.get("autopkg_prefs")))
+
+ repo_push_branch = config.Git.get("repo_push_branch")
+ stashed = False
+
+ try:
+
+ private_repo = git.Repo(os.path.expanduser(config.Git.get("local_repo_dir")))
+
+ if private_repo.is_dirty():
+ _ = private_repo.git.stash()
+ stashed = True
+
+ active_branch = private_repo.active_branch
+ local_branches = [ branch.name for branch in private_repo.branches ]
+
+ if repo_push_branch not in local_branches:
+ _ = private_repo.git.branch(repo_push_branch)
+
+ if repo_push_branch != active_branch:
+ _ = private_repo.git.checkout(repo_push_branch)
+
+ cmd = f"{config.AutoPkg.get('binary')} update-trust-info {recipe_id} {autopkg_options}"
+
+ if task_utils.get_user_context():
+ cmd = f"su - {task_utils.get_console_user()} -c \"{cmd}\""
+
+ # log.debug(f"Command to execute: {cmd}")
+ results = utility.execute_process(cmd)
+
+ if results["success"] and private_repo.git.diff():
+
+ log.info(f"Successfully updated trust for: {recipe_id}")
+ recipe_file_path = results["stdout"].split("Wrote updated ")[-1]
+ # log.debug(f"Updated recipe filename: {recipe_file_path}")
+ # Stage recipe_file_path
+ _ = private_repo.index.add([recipe_file_path])
+
+ _ = private_repo.git.commit(
+ "--message", "Updated Trust Info", "--message",
+ f"By: {config.Slack.get('bot_name')}"
+ )
+
+ _ = private_repo.git.push("--set-upstream", "origin", repo_push_branch)
+ log.info("Successfully updated private repo")
+
+ except Exception as error:
+ log.error(f"Failed to updated private repo due to:\n{error}")
+ results = {
+ "success": False,
+ "stdout": error,
+ "stderr": error
+ }
+
+ if stashed:
+ private_repo.git.stash("pop")
+
+ send_webhook.apply_async((self.request.id,), queue='autopkg', priority=9)
+
+ return {
+ "event": "update_trust_info",
+ "event_id": trust_id,
+ "recipe_id": recipe_id,
+ "success": results["success"],
+ "stdout": results["stdout"],
+ "stderr": results["stderr"],
+ }
diff --git a/pkgbot/tasks/task_utils.py b/pkgbot/tasks/task_utils.py
new file mode 100644
index 0000000..5ec57b0
--- /dev/null
+++ b/pkgbot/tasks/task_utils.py
@@ -0,0 +1,102 @@
+import os
+import re
+
+from datetime import datetime, timedelta
+
+from celery.result import AsyncResult
+
+from pkgbot import config
+from pkgbot.utilities import common as utility
+
+
+config = config.load_config()
+
+
+def get_task_results(task_id):
+ """ Return task info for the given task_id """
+
+ return AsyncResult(task_id)
+
+
+def get_user_context():
+
+ return os.getlogin() == "root" and os.getenv("USER") is None
+
+
+def get_console_user():
+
+ # Get the Console User
+ results_console_user = utility.execute_process(
+ "/usr/sbin/scutil", "show State:/Users/ConsoleUser")
+ return re.sub(
+ "(Name : )|(\n)", "", ( re.search("Name : .*\n", results_console_user["stdout"])[0] ))
+
+
+def check_recipe_schedule(interval, last_ran):
+ """Check if a recipe should be ran, based on the configured schedule.
+
+ Args:
+ interval (int): The "schedule" in number of days to not for
+ last_ran (str): datetime object in str format when recipe was last ran
+
+ Returns:
+ boolean:
+ True: Recipe should be ran
+ False: Recipe should not be ran
+ """
+
+ if interval != 0 and last_ran != None:
+ current_time = utility.utc_to_local(datetime.now())
+ last_ran_time = datetime.fromisoformat(last_ran)
+ interval_in_hours = interval * 24
+ return current_time - last_ran_time > timedelta(hours=interval_in_hours)
+
+ return True
+
+
+def api_url_helper():
+ secure = "s" if config.PkgBot.get("enable_ssl") else ""
+ pkgbot_server = f"http{secure}://{config.PkgBot.get('host')}:{config.PkgBot.get('port')}"
+ headers = { "Content-Type": "application/json" }
+ return pkgbot_server, headers
+
+
+def generate_autopkg_args(**kwargs):
+
+ options = ""
+
+ # AutoPkg args
+ if kwargs.get("verbose"):
+ options = f"{options} -{kwargs.get('verbose')}"
+
+ if kwargs.get("ignore_parent_trust"):
+ options = f"{options} --ignore-parent-trust-verification-errors"
+
+ if kwargs.get("prefs"):
+ options = f"{options} --prefs=\'{kwargs.get('prefs')}\'"
+ else:
+ options = f"{options} --prefs=\'{os.path.abspath(config.JamfPro_Dev.get('autopkg_prefs'))}\'"
+
+ # PkgBot args
+ if kwargs.get("promote_recipe_id"):
+ options = f"{options} --key \'RECIPE_ID={kwargs.get('promote_recipe_id')}\'"
+
+ if kwargs.get("match_pkg"):
+ options = f"{options} --key \'MATCH_PKG={kwargs.get('match_pkg')}\'"
+
+ if kwargs.get("pkg_only"):
+ options = f"{options} --key \'PKG_ONLY=True\'"
+
+ return options.lstrip()
+
+
+def compare_branch_heads(repo, local_branch, remote_branch):
+
+ return [
+ int(x) for x in (
+ repo.git.rev_list(
+ "--left-right",
+ "--count",
+ f"{local_branch}...{remote_branch}@{{u}}"
+ )).split('\t')
+ ]
diff --git a/templates/base.html b/pkgbot/templates/base.html
similarity index 100%
rename from templates/base.html
rename to pkgbot/templates/base.html
diff --git a/pkgbot/templates/index.html b/pkgbot/templates/index.html
new file mode 100644
index 0000000..fff309d
--- /dev/null
+++ b/pkgbot/templates/index.html
@@ -0,0 +1,26 @@
+{% extends 'base.html' %}
+
+{% block title %}Login{% endblock %}
+
+{% block content %}
+
+ {% if session["protected_page"] %}
+
+
+
×
+
Access Denied: You must login before accessing this page!
+
PkgBot: Systems Online
+
+
PkgBot will record software version history for Jamf Pro.
+
+PkgBot is an automation framework for the open source project AutoPkg that provides a web-based front end and a Slack Bot to send notifications and receive commands. It helps manage the lifecycle of software packaging through package and version validation and then provides an interactive method to "promote" a specific package version from "development" (or "test") to production environments.
+
+PkgBot provides this workflow utilizing Jamf Pro and the JamfUploader line of Processors. A Slack Bot is used to send new build notifications and allows a PkgBot Admin to interact with those notifications.
+
+This site provides visibility into third party software version history including each packages' current status, plus details on the available AutoPkg recipes.
+
+{% endblock %}
\ No newline at end of file
diff --git a/templates/login.html b/pkgbot/templates/login.html
similarity index 100%
rename from templates/login.html
rename to pkgbot/templates/login.html
diff --git a/templates/package.html b/pkgbot/templates/package.html
similarity index 100%
rename from templates/package.html
rename to pkgbot/templates/package.html
diff --git a/templates/packages.html b/pkgbot/templates/packages.html
similarity index 81%
rename from templates/packages.html
rename to pkgbot/templates/packages.html
index 5e24c96..ff306b5 100644
--- a/templates/packages.html
+++ b/pkgbot/templates/packages.html
@@ -55,18 +55,21 @@
{% set sort_on = request.sort_on | default("id") %}
{% for app in packages | sort(attribute=sort_on) %}
+ {% if app["icon"] %}
+  }}) |
+ {% elif app["jps_url"] and app["icon_id"] %}
 |
- {{ app.id }} |
-
+ {% else %}
+  }}) |
+ {% endif %}
+ {{ app.id }} |
{{ app.name }} |
{{ app.version }} |
{{ app.status }} |
{{ app.status_updated_by }} |
{{ app.packaged_date | strftime }} |
{{ app.promoted_date | strftime }} |
-
- {{ app.do_not_delete }} |
-
+ {{ app.common }} |
{{ app.special_flags }} |
{{ app.notes }} |
diff --git a/templates/recipe.html b/pkgbot/templates/recipe.html
similarity index 100%
rename from templates/recipe.html
rename to pkgbot/templates/recipe.html
diff --git a/templates/recipes.html b/pkgbot/templates/recipes.html
similarity index 92%
rename from templates/recipes.html
rename to pkgbot/templates/recipes.html
index 0d10d5a..2b5bd3b 100644
--- a/templates/recipes.html
+++ b/pkgbot/templates/recipes.html
@@ -55,10 +55,11 @@
{% set sort_on = request.sort_on | default("id") %}
{% for recipe in recipes | sort(attribute=sort_on) %}
- | {{ recipe.id }} |
+ {{ recipe.id }} |
{{ recipe.recipe_id }} |
{{ recipe.name }} |
{{ recipe.enabled }} |
+ {{ recipe.manual_only }} |
{{ recipe.pkg_only }} |
{{ recipe.last_ran | strftime }} |
{{ recipe.schedule }} |
diff --git a/pkgbot/utilities/__init__.py b/pkgbot/utilities/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pkgbot/utilities/celery.py b/pkgbot/utilities/celery.py
new file mode 100644
index 0000000..b7bdbef
--- /dev/null
+++ b/pkgbot/utilities/celery.py
@@ -0,0 +1,21 @@
+from celery import current_app as current_celery_app
+
+from pkgbot import settings
+
+
+def create_celery():
+ celery_app = current_celery_app
+ celery_app.config_from_object(settings.celery.settings, namespace='CELERY')
+ celery_app.conf.update(task_track_started=True)
+ celery_app.conf.update(task_serializer='pickle')
+ celery_app.conf.update(result_serializer='pickle')
+ celery_app.conf.update(accept_content=['pickle', 'json'])
+ celery_app.conf.update(result_expires=200)
+ celery_app.conf.update(result_persistent=True)
+ celery_app.conf.update(worker_send_task_events=False)
+ celery_app.conf.update(worker_prefetch_multiplier=1)
+
+ return celery_app
+
+
+##### This file currently isn't used
diff --git a/pkgbot/utilities/common.py b/pkgbot/utilities/common.py
new file mode 100755
index 0000000..bf3cb04
--- /dev/null
+++ b/pkgbot/utilities/common.py
@@ -0,0 +1,300 @@
+import asyncio
+import hashlib
+import hmac
+import logging.config
+import os
+# import pickle
+import plistlib
+import re
+import shlex
+import subprocess
+import yaml
+
+from datetime import datetime, timezone
+from distutils.util import strtobool
+
+# from sqlalchemy import create_engine
+# from sqlalchemy.orm import sessionmaker
+
+from pkgbot import config
+
+
+config = config.load_config()
+
+
+def log_setup(name="PkgBot"):
+
+ logger = logging.getLogger(name)
+
+ if not logger.hasHandlers():
+ logger.debug("LOGGER HAS NO HANDLERS!")
+
+ # Get the log configuration
+ log_config = yaml.safe_load(f"{config.PkgBot.get('log_config')}")
+
+ # Load log configuration
+ logging.config.dictConfig(log_config)
+
+ else:
+ logger.debug("Logger has handlers!")
+
+ # Create logger
+ return logger
+
+
+log = log_setup()
+
+
+async def run_process_async(command, input=None):
+ """
+ A helper function for asyncio's subprocess.
+
+ Args:
+ command (str): The command line level syntax that would be
+ written in shell or a terminal window.
+ Returns:
+ Results in a dictionary.
+ """
+
+ # Validate that command is not a string
+ if not isinstance(command, str):
+ raise TypeError('Command must be a str type')
+
+ # Format the command
+ # command = shlex.quote(command)
+
+ # Run the command
+ process = await asyncio.create_subprocess_shell(
+ command, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE)
+
+ if input:
+ (stdout, stderr) = await process.communicate(input=bytes(input, "utf-8"))
+ else:
+ (stdout, stderr) = await process.communicate()
+
+ return {
+ "stdout": (stdout.decode()).strip(),
+ "stderr": (stderr.decode()).strip() if stderr != None else None,
+ "status": process.returncode,
+ "success": True if process.returncode == 0 else False
+ }
+
+
+def execute_process(command, input=None):
+ """
+ A helper function for subprocess.
+
+ Args:
+ command (str): The command line level syntax that would be
+ written in shell or a terminal window.
+ Returns:
+ Results in a dictionary.
+ """
+
+ # Validate that command is not a string
+ if not isinstance(command, str):
+ raise TypeError('Command must be a str type')
+
+ # Format the command
+ # command = shlex.quote(command)
+
+ # Run the command
+ process = subprocess.Popen(
+ command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
+ if input:
+ (stdout, stderr) = process.communicate(input=bytes(input, "utf-8"))
+ else:
+ (stdout, stderr) = process.communicate()
+
+ return {
+ "stdout": (stdout.decode()).strip(),
+ "stderr": (stderr.decode()).strip() if stderr != None else None,
+ "status": process.returncode,
+ "success": True if process.returncode == 0 else False
+ }
+
+
+async def ask_yes_or_no(question):
+ """Ask a yes/no question via input() and determine the value of the answer.
+
+ Args:
+ question: A string that is written to stdout
+
+ Returns:
+ True of false based on the users' answer.
+
+ """
+
+ print(f"{question} [Yes/No] ", end="")
+
+ while True:
+ try:
+ return strtobool(input().lower())
+ except ValueError:
+ print("Please respond with [yes|y] or [no|n]: ", end="")
+
+
+async def plist_reader(plistFile):
+ """A helper function to get the contents of a Property List.
+ Args:
+ plistFile: A .plist file to read in.
+ Returns:
+ stdout: Returns the contents of the plist file.
+ """
+
+ if os.path.exists(plistFile):
+ with open(plistFile, "rb") as plist:
+ plist_contents = plistlib.load(plist)
+ return plist_contents
+
+
+async def utc_to_local(utc_dt):
+
+ return utc_dt.replace(tzinfo=timezone.utc).astimezone(tz=None)
+
+
+async def string_to_datetime(datetime_string: str, format_string: str = "%Y-%m-%d %H:%M:%S.%f"):
+
+ return datetime.strptime(datetime_string, format_string)
+
+
+async def datetime_to_string(datetime_string: str, format_string: str = "%Y-%m-%d %I:%M:%S"):
+
+ converted = datetime.fromisoformat(datetime_string)
+ return converted.strftime(format_string)
+
+
+async def compute_hex_digest(key: bytes,
+ message: bytes, hash: hashlib._hashlib.HASH = hashlib.sha256):
+
+ return hmac.new(key, message, hash).hexdigest()
+
+
+async def load_yaml(config_file):
+
+ # Load the recipe config
+ with open(config_file, 'rb') as config_file_path:
+ return yaml.safe_load(config_file_path)
+
+
+async def save_yaml(contents, config_file):
+ """Writes the passed dict to the passed file.
+
+ Args:
+ contents (dict): a updated dict object of recipes
+ config_file (str): path to the configuration file to update
+ """
+
+ with open(config_file, 'w', encoding="utf8") as config_file_path:
+ yaml.dump(contents, config_file_path)
+
+
+async def replace_sensitive_strings(message, sensitive_strings=None, sensitive_regex_strings=None):
+ """Redact sensitive strings, such as passwords, serial numbers, license keys, etc. before
+ exporting to a non-secure location.
+
+ Args:
+ message (str, Any): A message that could contain sensitive strings. If `message` is not a
+ string, it will be "converted" to a string via `str(message)`.
+ sensitive_strings (str, optional): A string of sensitive strings, separated by a `|` (pipe).
+ These strings will be Regex escaped. Defaults to None.
+ sensitive_regex_strings (str, optional): A string of sensitive strings in Regex format,
+ separated by a `|` (pipe). Defaults to None.
+ """
+
+
+ async def parse_for_sensitive_keys(a_dict, sensitive_key_names):
+
+ found_sensitive_strings = ""
+
+ for key, value in a_dict.items():
+
+ if re.search(rf".*({sensitive_key_names}).*", key, re.IGNORECASE) and value:
+
+ if found_sensitive_strings:
+ found_sensitive_strings = "|".join([found_sensitive_strings, re.escape(value)])
+ else:
+ found_sensitive_strings = re.escape(value)
+
+ return found_sensitive_strings
+
+
+ all_sensitive_strings = r"bearer\s[\w+.-]+|"
+ sensitive_key_names = r"password|secret|license|serial|key"
+
+ if config.Common.get("additional_sensitive_key_names"):
+ sensitive_key_names += f"|{config.Common.get('additional_sensitive_key_names')}"
+
+ for plist in [
+ config.JamfPro_Prod.get("autopkg_prefs"),
+ config.JamfPro_Dev.get("autopkg_prefs")
+ ]:
+ plist_contents = await plist_reader(plist)
+ all_sensitive_strings += await parse_for_sensitive_keys(plist_contents, sensitive_key_names)
+
+ for string in [
+ config.Common.get("redaction_strings"),
+ sensitive_regex_strings
+ ]:
+ if string:
+ all_sensitive_strings = "|".join([all_sensitive_strings, string])
+
+ for string in [
+ config.JamfPro_Dev.get("api_user"),
+ config.JamfPro_Dev.get("api_password"),
+ config.JamfPro_Dev.get("dp1_user"),
+ config.JamfPro_Dev.get("dp1_password"),
+ config.JamfPro_Prod.get("api_user"),
+ config.JamfPro_Prod.get("api_password"),
+ config.JamfPro_Prod.get("dp1_user"),
+ config.JamfPro_Prod.get("dp1_password"),
+ sensitive_strings
+ ]:
+ if string:
+ all_sensitive_strings = "|".join([all_sensitive_strings, re.escape(string)])
+
+ if isinstance(message, str):
+ return re.sub(rf"{all_sensitive_strings}", '', message)
+
+ elif isinstance(message, dict):
+
+ for key, value in message.items():
+ if not isinstance(value, (bool, int)) and value is not None:
+ message[key] = re.sub(rf"{all_sensitive_strings}", '', value)
+
+ return message
+
+ else:
+ log.warning(
+ f"Unaccounted for type in sensitive string substitution! Type is: {type(message)}")
+ return re.sub(rf"{all_sensitive_strings}", '', str(message))
+
+
+# async def get_task_results(task_id: str):
+
+# # https://docs.sqlalchemy.org/en/14/core/engines.html#sqlite
+# db_engine = create_engine(f"sqlite:///{config.Database.get('location')}")
+# Session = sessionmaker(db_engine)
+
+# with Session() as session:
+# result = session.execute(f"SELECT result from celery_taskmeta where task_id = '{task_id}';").fetchone()
+
+# return pickle.loads(result.result)
+
+
+async def find_receipt_plist(content: str):
+
+ run_receipt = re.search(r'Receipt written to (.*)', content)[1]
+ return await plist_reader(run_receipt)
+
+
+async def parse_recipe_receipt(content: dict, key: str):
+
+ for step in reversed(content):
+ if step.get(key):
+ return step.get(key)
+ elif re.search(key, step.get("Processor"), re.IGNORECASE):
+ return step
diff --git a/requirements.txt b/requirements.txt
old mode 100644
new mode 100755
index db80a1e..358f313
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,37 +1,56 @@
-aiohttp==3.7.4.post0
-aiosqlite==0.16.1
-anyio==3.2.1
-async-timeout==3.0.1
-attrs==20.3.0
-certifi==2022.5.18.1
+aiohttp==3.8.1
+aiosignal==1.2.0
+aiosqlite==0.17.0
+amqp==5.1.1
+anyio==3.6.1
+async-timeout==4.0.2
+billiard==3.6.4.0
+celery==5.2.7
chardet==4.0.0
-click==7.1.2
-fastapi==0.63.0
-fastapi-login==1.6.0
+charset-normalizer==2.0.12
+click-didyoumean==0.3.0
+click-plugins==1.1.1
+click-repl==0.2.0
+fastapi==0.78.0
+fastapi-login==1.8.2
+fastapi-utils==0.2.1
+flower==1.0.0
+frozenlist==1.3.0
+gitdb==4.0.9
+GitPython==3.1.27
h11==0.12.0
-httpcore==0.13.6
-httpx==0.18.2
-idna==2.10
-iso8601==0.1.14
-Jinja2==2.11.3
-MarkupSafe==1.1.1
-multidict==5.1.0
+httpcore==0.15.0
+httpx==0.23.0
+humanize==4.2.1
+idna==3.3
+iso8601==1.0.2
+Jinja2==3.1.2
+kombu==5.2.4
+MarkupSafe==2.1.1
+multidict==6.0.2
passlib==1.7.4
-pydantic==1.8.1
-PyJWT==2.1.0
+prometheus-client==0.14.1
+prompt-toolkit==3.0.29
+pydantic==1.9.1
+PyJWT==2.4.0
pypika-tortoise==0.1.5
python-multipart==0.0.5
-pytz==2020.5
-PyYAML==5.4.1
-requests==2.25.1
+pytz==2022.1
+PyYAML==6.0
+requests==2.28.0
rfc3986==1.5.0
secure==0.3.0
six==1.16.0
-slack-sdk==3.4.2
+slack-sdk==3.17.2
+smmap==5.0.0
sniffio==1.2.0
-starlette==0.13.6
-tortoise-orm==0.17.3
-typing-extensions==3.7.4.3
-urllib3==1.26.4
-uvicorn==0.13.4
-yarl==1.6.3
+SQLAlchemy==1.4.39
+starlette==0.19.1
+tornado==6.1
+tortoise-orm==0.19.1
+typing-extensions==4.2.0
+urllib3==1.26.9
+uvicorn==0.18.1
+vine==5.0.0
+wcwidth==0.2.5
+yarl==1.7.2
diff --git a/templates/index.html b/templates/index.html
deleted file mode 100644
index 2619e4e..0000000
--- a/templates/index.html
+++ /dev/null
@@ -1,26 +0,0 @@
-{% extends 'base.html' %}
-
-{% block title %}Login{% endblock %}
-
-{% block content %}
-
- {% if session["protected_page"] %}
-
-
-
×
-
Access Denied: You must login before accessing this page!
-
PkgBot: Systems Online
-
-
PkgBot will record software version history for Jamf Pro.
-
-PkgBot is a framework to manage software packaging, testing, and promoting from a development to production environment. It utilizes the open source project AutoPkg to download and package software. From there, the JSSImporter Processor is utilized to upload packages into Jamf Pro. Several customizations have been made to provide the functionality that PkgBot offers.
-
-Within this site, you will find software version history including it's current status, as well as AutoPkg recipe configurations and status.
-
-This site will provide Site Admins greater visibility into third party software updates and more in the future. We hope this project promotes more collaboration with Deskside groups to validate software .pkgs before they're made available in production and customer systems.
-
-{% endblock %}
\ No newline at end of file
diff --git a/utils.py b/utils.py
deleted file mode 100644
index 1e39232..0000000
--- a/utils.py
+++ /dev/null
@@ -1,184 +0,0 @@
-#!/usr/local/autopkg/python
-
-import asyncio
-import hashlib
-import hmac
-import logging.config
-import os
-import plistlib
-import re
-import shlex
-import subprocess
-import yaml
-
-from datetime import datetime, timezone, tzinfo
-from distutils.util import strtobool
-
-import config
-
-
-config.load()
-
-
-def log_setup(name="PkgBot"):
-
- logger = logging.getLogger(name)
-
- if not logger.hasHandlers():
- logger.debug("LOGGER HAS NO HANDLERS!")
-
- # Get the log configuration
- log_config = yaml.safe_load("{}".format(config.pkgbot_config.get("PkgBot.log_config")))
-
- # Load log configuration
- logging.config.dictConfig(log_config)
-
- else:
- logger.debug("Logger has handlers!")
-
- # Create logger
- return logger
-
-
-log = log_setup()
-
-
-async def run_process_async(command, input=None):
- """
- A helper function for asyncio's subprocess.
-
- Args:
- command: The command line level syntax that would be
- written in shell or a terminal window. (str)
- Returns:
- Results in a dictionary.
- """
-
- # Validate that command is not a string
- if not isinstance(command, str):
- raise TypeError('Command must be a str type')
-
- # Format the command
- # command = shlex.quote(command)
-
- # Run the command
- process = await asyncio.create_subprocess_shell(
- command, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE)
-
- if input:
- (stdout, stderr) = await process.communicate(input=bytes(input, "utf-8"))
-
- else:
- (stdout, stderr) = await process.communicate()
-
- return {
- "stdout": (stdout.decode()).strip(),
- "stderr": (stderr.decode()).strip() if stderr != None else None,
- "status": process.returncode,
- "success": True if process.returncode == 0 else False
- }
-
-
-async def ask_yes_or_no(question):
- """Ask a yes/no question via input() and determine the value of the answer.
-
- Args:
- question: A string that is written to stdout
-
- Returns:
- True of false based on the users' answer.
-
- """
-
- print("{} [Yes/No] ".format(question), end="")
-
- while True:
-
- try:
- return strtobool(input().lower())
-
- except ValueError:
- print("Please respond with [yes|y] or [no|n]: ", end="")
-
-
-async def plist_reader(plistFile):
- """A helper function to get the contents of a Property List.
- Args:
- plistFile: A .plist file to read in.
- Returns:
- stdout: Returns the contents of the plist file.
- """
-
- if os.path.exists(plistFile):
-
- with open(plistFile, "rb") as plist:
-
- plist_contents = plistlib.load(plist)
-
- return plist_Contents
-
-
-async def utc_to_local(utc_dt):
-
- return utc_dt.replace(tzinfo=timezone.utc).astimezone(tz=None)
-
-
-async def string_to_datetime(datetime_string: str, format_string: str = None):
-
- if not format_string:
- format_string = "%Y-%m-%d %H:%M:%S.%f"
-
- return datetime.strptime(datetime_string, format_string)
-
-
-async def datetime_to_string(datetime_string: str, format_string: str = None):
-
- if not format_string:
- format_string = "%Y-%m-%d %I:%M:%S"
-
- converted = datetime.fromisoformat(str(datetime_string))
-
- return converted.strftime(format_string)
-
-
-async def compute_hex_digest(key: bytes, message: bytes):
-
- return hmac.new( key, message, hashlib.sha256 ).hexdigest()
-
-
-async def load_yaml(config_file):
-
- # Load the recipe config
- with open(config_file, 'rb') as config_file_path:
- return yaml.safe_load(config_file_path)
-
-
-async def save_yaml(contents, config_file):
- """Writes the passed dict to the passed file.
-
- Args:
- contents (dict): a updated dict object of recipes
- config_file (str): path to the configuration file to update
- """
-
- with open(config_file, 'w', encoding="utf8") as config_file_path:
- yaml.dump(contents, config_file_path)
-
-
-async def replace_sensitive_strings(message, sensitive_strings=None):
-
- default_sensitive_strings = r'{}|{}|{}|{}|{}|{}'.format(
- config.pkgbot_config.get("JamfPro_Prod.api_password"),
- config.pkgbot_config.get("JamfPro_Prod.api_user"),
- config.pkgbot_config.get("JamfPro_Prod.dp1_user"),
- config.pkgbot_config.get("JamfPro_Prod.dp1_password"),
- config.pkgbot_config.get("Common.RedactionStrings"),
- r"bearer\s[\w+.-]+"
- )
-
- if sensitive_strings:
-
- default_sensitive_strings = "{}|{}".format(default_sensitive_strings, sensitive_strings)
-
- return re.sub(default_sensitive_strings, '', message)
## About
-PkgBot is currently written to support this workflow utilizing Jamf Pro and this build specifically expects the use of the JSSImporter Processor to upload packages into Jamf Pro. From there, several customizations are were made to `JSSImporter` as will as Post Processors, to provide the functionality that PkgBot offers. This build will be the last to support `JSSImporter` and PkgBot receive a massive overhaul in the near future.
+PkgBot provides this workflow utilizing Jamf Pro and the [JamfUploader](https://github.com/grahampugh/jamf-upload) line of Processors. A Slack Bot is used to send new build notifications and allows a `PkgBot Admin` to interact with those notifications.
-A Slack Bot is used to send new build notifications and allows a `PkgBot Admin` to interact with those notifications. AutoPkg generates notifications via a custom Post Processor aptly named "PkgBot." This Post Processor communicates with the PkgBot API to send messages to Slack.
+To "promote" a package to a production Jamf Pro instance without re-running the entire recipe chain, a custom Post Processor (inspired by Graham Pugh's [JSSRecipeReceiptChecker](https://github.com/autopkg/grahampugh-recipes/blob/master/CommonProcessors/JSSRecipeReceiptChecker.py)) is used find and acquire the matching recipe dev run details. The values are passed to a "production recipe template" that performs the JamfUploader steps which can be configured to upload the package and optionally, update other various items (e.g. Policy, Group, Scripts, etc.) _without_ re-downloading nor re-packaging.
-To "promote" a package to a production Jamf Pro instance, a custom Post Processor (forked and heavily customized from Graham Pugh's [JSSRecipeReceiptChecker](https://github.com/autopkg/grahampugh-recipes/blob/master/CommonProcessors/JSSRecipeReceiptChecker.py)) along with a custom "recipe template" is used find the previous autopkg run details, get the required info and upload the package and, optionally, update other various items (e.g. Policy, Group, Scripts, etc.) _without_ re-downloading nor re-packaging.
+A web-based front end is available to review the status and history of all packages as well as the _known_ `AutoPkg` recipe configurations and statuses.
-A web view is also provided where all package status and history can be reviewed as well as `AutoPkg` recipe configurations and status.
+PkgBot has been running in my production environment for over a year now and is working quite well. I've been ironing out the kinks and making improvements to the overall processes and workflows to streamline everything.
-PkgBot has been running in my production environment for a little over a year now and working quite well. I've been ironing out some of the kinks and making improvements to the overall process and workflows as I attempt to streamline everything.
-
-
## Backend Design
-This project is built around FastAPI and several other core libraries. I built it with an API workflow similar to that of the Jamf Pro and Classic APIs in-mind. So, if you've worked with these, this project will have a familiar feel.
+This project is built around FastAPI, Celery, the Slack SDK, and several other core libraries. I built it with an API similar to the Jamf Pro (UAPI) and Classic APIs. So if you've worked with these, this project's API will have a familiar feel.
-The project has a fully async code base and utilizes numerous popular Python libraries.
+The project has a fully asynchronous code base and utilizes numerous popular Python libraries.
-
## Planned Features
- * Moving to a proper "backend" system for executing tasks
+ * ~~Moving to a proper "backend" system for executing tasks~~
* (More) Streamlining (of) workflows
- * "Hosting" the icons within PkgBot instead of Jamf Pro
+ * ~~"Hosting" the icons within PkgBot instead of Jamf Pro~~
* Slack slash commands for executing recipes
* Support for "cleaning up" old notifications
* e.g. when an app version has been "retired"
* Code Improvements
- * Better config loading
- * Better log loading/usage
+ * ~~Better config loading~~
+ * ~~Better log loading/usage~~
* A "setup/install" script
-
## Requirements
-PkgBot will be written to support the Python3 framework that is shipped with `AutoPkg`. It does need numerous additional libraries that will have to be installed separately that are not included with `AutoPkg`'s bundled Python3.
+PkgBot will be written to support the Python3 framework that is shipped with `AutoPkg` (currently supporting AutoPkg 2.6.0's bundled Python 3.10). It needs additional libraries that are not included with `AutoPkg`'s bundled Python3 that need to be installed separately. It also requires RabbitMQ.
-The major libraries are:
+The major Python libraries are:
* FastAPI
+ * Celery
* Jinja2
- * For the web views portion
+ * For the web front-end
* Slack SDK
* For Notifications
* Tortoise ORM
@@ -61,137 +60,25 @@ The major libraries are:
See the requirements.txt file for additional libraries and dependencies.
-## ("_Basic_") How to Setup
-
-Below will be the basics to get this setup and working. A few more bits and pieces can be done to customize things further. (For example, a LaunchDaemon to run the PkgBot webserver and a LaunchAgent to kick off scheduled runs of autopkg. Samples will be provided in the extras directory.)
-
-1. Install the prerequisites:
- * Git
- * AutoPkg
- * JSSImporter
-
-2. Clone this repo and store it on your AutoPkg Runner.
- * `git clone https://github.com/mlbz521/PkgBot.git "/Library/AutoPkg/"`
-
-3. Install requirements in specific location
- * e.g. `pip install --target="/Library/AutoPkg/PkgBotPackages" -r requirements.txt`
- * Or, if your simply testing, create and active a virtual environment and install the requirements
-
-4. Create a Slack Bot/App
- * There are numerous tutorials on how to do this and I'm not going to go over the entire process here. I will simply provide the configuration requirements. [Official documentation](https://slack.com/help/articles/115005265703-Create-a-bot-for-your-workspace)
- * Features/Functionality required
- * Incoming Webhooks
- * Create a webhook to post to the desired channel
- * Interactive Components
- * Set a `Request URL` that the Bot will send messages too and your server will receive on
- * e.g. `https://pkgbot.my.server.org/slackbot/receive`
- * or, if using ngrok: `https://84c5df439d74.ngrok.io/slackbot/receive` (see below)
- * Bots
- * OAuth & Permissions
- * Scopes
- * Bot Token Scopes
- * chat:write
- * files:write
- * reactions:read
- * reactions:write
- * incoming-webhook
- * Tokens/Secrets/Keys required:
- * Bot User OAuth Token
- * Signing Secret
- * Bot Name
- * Channel
- * Channel it will be posting into
- * Note: You can test PkgBot without creating the SlackBot -- obviously expect for the _actual_ Slack notifications part.
-
-5. Configure your environments' settings (`/[path/to/PkgBot]/settings/pkgbot_config.yaml`).
-
-6. Ensure your PkgBot "server" can communicate with Slack's API
- * For testing, you can utilize ngrok to allow communication from Slack to your dev box.
- * There are numerous tutorials on how to do this and I'm not going to go over the entire process here. I will simply provide the configuration requirements. [Official documentation ](https://ngrok.com/docs/getting-started)
- * Follow steps two through four above
- * In step four, use the same port defined in your `pkgbot_config.yaml` file
- * e.g. `ngrok http 443`
- * After starting ngrok, grab the forwarding address from the console
- * e.g. `Forwarding https://84c5df439d74.ngrok.io -> http://localhost:443`
- * the forwarding address is: `https://84c5df439d74.ngrok.io`
- * The forwarding address will need to be entered into your Slack Bot configuration
-
-7. Optionally, create a private/public certificate for use with Uvicorn (_not required when testing with ngrok_)
- * Generate a private key and a csr:
- * `openssl req -new -newkey rsa:2048 -nodes -keyout private.key -out pkgbot_csr.csr`
- * Get a publicly trusted cert with using the CSR
- * Update the pkgbot_config with these values
-
-8. Run PkgBot via:
- * `python3 pkgbot.py`
- * or `chmod +x pkgbot.py && pkgbot.py`
-
-
-### "_Basic_" Examples
-
-Example command to run via `autopkg` directly with the PkgBot Post Processor
-
-`/usr/local/bin/autopkg run com.github.mlbz521.jss.ProductionTemplate --key recipe_id="com.github.mlbz521.jss.Dropbox" --prefs="/Library/AutoPkg/PkgBotServer/settings/prod_autopkg_prefs.plist" --postprocessor PkgBot -vv`
-
-#### Using the pseudo cli tool
-
-How to import recipes from a yaml file
-
-`python3 -m execute.autopkg manage import --input ./settings/recipe_config.yaml`
-```
-2021-06-25 15:32:55,781 - PkgBot - INFO - Importing recipe config file from: ./settings/recipe_config.yaml.
-2021-06-25 15:33:52,474 - PkgBot - INFO - All recipe configurations have been imported!
-```
-
-Manage a single recipe
-
-`python3 -m execute.autopkg manage single -i com.github.mlbz521.jss.Zoom-ForIT --enable --force`
-
-Run a single recipe via PkgBot
-
-`python3 -m execute.autopkg run -e dev -i com.github.mlbz521.jss.Brother-MFC-J6935DW --pkgbot_config "/Library/AutoPkg/PkgBotServer/settings/pkgbot_config.yaml"`
-
-
-#### Managing the LaunchAgent for Kicking off AutoPkg runs
-
-Load launchagent to run recipes
-
-`launchctl bootstrap "gui/501" ~/Library/LaunchAgents/com.github.mlbz521.autopkg.service.plist`
-
-Kickstart run of recipes
-
-`launchctl kickstart -p "gui/501/com.github.mlbz521.autopkg.service"`
-
-Stop
-
-`launchctl bootout "gui/501/com.github.mlbz521.autopkg.service"`
-
-
-#### Managing the PkgBot LaunchDaemon
-
-Start PkgBot
-
-`sudo launchctl bootstrap system /Library/LaunchDaemons/com.github.mlbz521.pkgbot.service.plist`
-
-Stop
+## How to Setup
-`sudo launchctl bootout "system/com.github.mlbz521.pkgbot.service"`
+The basics to get PkgBot setup and working are covered in the [Wiki](https://github.com/MLBZ521/PkgBot/wiki/%22Basic%22-How-to-Setup).
## The Why(s) and My Thought Process
-At my organization, I have moved almost everything that needs to be "packaged" to be _packaged_ by AutoPkg. This was done for numerous reasons:
+For my organization, almost every applications that needs to be "packaged," I have AutoPkg _packaging_ it. This was done for numerous reasons:
* automation
- * the recipe creates documentation of the steps to reproduce a package
+ * the recipe chain is self documenting of the steps to reproduce a package
* e.g. for a new version
- * a single "this is the way" [_that we do things_]
+ * a single "this is the way" (_that we do things_)
* easier for team members to pick up and go
* scalability
In addition, there are a large number of Site Admins in my organization and I'm always receiving questions like:
> "When is \