diff --git a/.github/workflows/deploy_to_prod.yml b/.github/workflows/deploy_to_prod.yml index 8a371b9f..f40cc221 100644 --- a/.github/workflows/deploy_to_prod.yml +++ b/.github/workflows/deploy_to_prod.yml @@ -8,6 +8,7 @@ on: jobs: deploy_to_prod: runs-on: ubuntu-latest + environment: production steps: - name: Checkout code @@ -49,13 +50,12 @@ jobs: }, "environment": { "GH_PRIVATE_KEY": "${{secrets.GH_PRIVATE_KEY}}", - "GITHUB_OWNER": "${{secrets.GH_OWNER}}", - "GITHUB_REPO": "${{secrets.GH_REPO}}", - "GITHUB_APP_ID": "${{secrets.GH_APP_ID}}", - "GITHUB_INSTALLATION_ID": "${{secrets.GH_INSTALLATION_ID}}", + "GITHUB_APP_ID": "${{vars.GH_APP_ID}}", "FILPLUS_ENV": "prod", "RUST_LOG": "debug", - "BOT_USER": "${{secrets.BOT_USER}}" + "BOT_USER": "${{vars.BOT_USER}}", + "BACKEND_URL": "${{vars.BACKEND_URL}}", + "DB_URL": "${{secrets.DB_URL}}" } } }' > containers.json diff --git a/.github/workflows/deploy_to_staging.yml b/.github/workflows/deploy_to_staging.yml index ecb13176..ce1d4db2 100644 --- a/.github/workflows/deploy_to_staging.yml +++ b/.github/workflows/deploy_to_staging.yml @@ -24,6 +24,7 @@ jobs: name: Deploy to staging needs: end_to_end_tests runs-on: ubuntu-latest + environment: staging steps: - name: Checkout code @@ -64,8 +65,14 @@ jobs: "8080": "HTTP" }, "environment": { - "GH_PRIVATE_KEY": "${{secrets.GH_STAGING_PRIVATE_KEY}}", - "DB_URL": "${{secrets.STAGING_DB_URL}}" + "GH_PRIVATE_KEY": "${{secrets.GH_PRIVATE_KEY}}", + "DB_URL": "${{secrets.DB_URL}}" + "GITHUB_APP_ID": "${{vars.GH_APP_ID}}", + "FILPLUS_ENV": "prod", + "RUST_LOG": "debug", + "BOT_USER": "${{vars.BOT_USER}}", + "BACKEND_URL": "${{vars.BACKEND_URL}}", + "DB_URL": "${{secrets.DB_URL}}" } } }' > containers.json diff --git a/fplus-http-server/src/router/allocator.rs b/fplus-http-server/src/router/allocator.rs index dcdfc070..1814f236 100644 --- a/fplus-http-server/src/router/allocator.rs +++ b/fplus-http-server/src/router/allocator.rs @@ -1,6 +1,6 @@ use actix_web::{get, post, put, delete, web, HttpResponse, Responder}; use fplus_database::database::allocators as allocators_db; -use fplus_lib::core::{allocator::process_allocator_file, AllocatorUpdateInfo, ChangedAllocator}; +use fplus_lib::core::{allocator::{process_allocator_file, is_allocator_repo_created, create_allocator_repo}, AllocatorUpdateInfo, ChangedAllocator}; /** * Get all allocators @@ -44,14 +44,29 @@ pub async fn create_from_json(file: web::Json) -> actix_web::R Some(model.application.verifiers_gh_handles.join(", ")) // Join verifiers in a string if exists }; - match allocators_db::create_or_update_allocator( - model.owner, - model.repo, + let allocator_model = match allocators_db::create_or_update_allocator( + model.owner.clone(), + model.repo.clone(), Some(model.installation_id as i64), Some(model.multisig_address), verifiers_gh_handles, ).await { - Ok(allocator_model) => Ok(HttpResponse::Ok().json(allocator_model)), + Ok(allocator_model) => allocator_model, + Err(e) => return Ok(HttpResponse::BadRequest().body(e.to_string())), + }; + + match is_allocator_repo_created(&model.owner, &model.repo).await { + Ok(true) => Ok(HttpResponse::Ok().json(allocator_model)), + Ok(false) => { + //Create allocator repo. If it fails, return http error + match create_allocator_repo(&model.owner, &model.repo).await { + Ok(files_list) => { + log::info!("Allocator repo created successfully: {:?}", files_list); + Ok(HttpResponse::Ok().json(allocator_model)) + } + Err(e) => Ok(HttpResponse::BadRequest().body(e.to_string())), + } + }, Err(e) => Ok(HttpResponse::BadRequest().body(e.to_string())), } }, diff --git a/fplus-lib/src/config.rs b/fplus-lib/src/config.rs index 680df884..c0d884e7 100644 --- a/fplus-lib/src/config.rs +++ b/fplus-lib/src/config.rs @@ -9,7 +9,7 @@ pub fn default_env_vars() -> &'static HashMap<&'static str, &'static str> { m.insert("GITHUB_OWNER", "filecoin-project"); m.insert("GITHUB_REPO", "filecoin-plus-falcon"); m.insert("GITHUB_APP_ID", "826129"); - m.insert("GITHUB_INSTALLATION_ID", "47207972"); + m.insert("GITHUB_INSTALLATION_ID", "48206379"); m.insert("RUST_LOG", "info"); m.insert("RUST_BACKTRACE", "1"); m.insert("DB_URL", ""); diff --git a/fplus-lib/src/core/allocator/mod.rs b/fplus-lib/src/core/allocator/mod.rs index ba3a8297..0d7c1993 100644 --- a/fplus-lib/src/core/allocator/mod.rs +++ b/fplus-lib/src/core/allocator/mod.rs @@ -1,7 +1,7 @@ use octocrab::models::repos::ContentItems; use crate::config::get_env_var_or_default; -use crate::external_services::github::github_async_new; +use crate::external_services::github::{github_async_new, GithubWrapper}; use crate::{base64::decode_allocator_model, error::LDNError}; use self::file::AllocatorModel; @@ -12,17 +12,17 @@ pub async fn process_allocator_file(file_name: &str) -> Result Result { let encoded_content = match file.items.get(0).and_then(|f| f.content.clone()) { Some(content) => { @@ -48,4 +48,89 @@ fn content_items_to_allocator_model(file: ContentItems) -> Result Result { + let repo_flag_file = "invalisd.md"; + let applications_directory = "applications"; + let gh = github_async_new(owner.to_string(), repo.to_string()).await; + let all_files_result = gh.get_files(applications_directory).await.map_err(|e| { + LDNError::Load(format!("Failed to retrieve all files from GitHub. Reason: {}", e)) + }); + + match all_files_result { + Ok(content_items) => { + let mut is_repo_created = false; + for file in content_items.items.iter() { + if file.name == repo_flag_file { + is_repo_created = true; + break; + } + } + Ok(is_repo_created) + }, + Err(e) => { + if e.to_string().contains("GitHub: This repository is empty") || e.to_string().contains("GitHub: Not Found"){ + Ok(false) + } else { + Err(e) + } + }, + } +} + +pub async fn create_allocator_repo(owner: &str, repo: &str) -> Result<(), LDNError> { + let gh = github_async_new(owner.to_string(), repo.to_string()).await; + let mut dirs = Vec::new(); + dirs.push("".to_string()); + + while dirs.len() > 0 { + let dir = dirs.pop().unwrap(); + let files_list = gh.get_files_from_public_repo("clriesco", "filplus-allocator-template", Some(&dir)).await.map_err(|e| { + LDNError::Load(format!("Failed to retrieve all files from GitHub. Reason: {}", e)) + })?; + + for file in files_list.items.iter() { + let file_path = file.path.clone(); + if file.r#type == "dir" { + dirs.push(file_path); + continue; + } + let file = reqwest::Client::new() + .get(&file.download_url.clone().unwrap()) + .send() + .await + .map_err(|e| LDNError::Load(format!("here {}", e)))?; + let file = file + .text() + .await + .map_err(|e| LDNError::Load(format!("here1 {}", e)))?; + + //Get file from target repo. If file does not exist or fails to retrieve, create it + let target_file = gh.get_file(&file_path, "main").await.map_err(|e| { + LDNError::Load(format!("Failed to retrieve file from GitHub. Reason: {} in file {}", e, file_path)) + }); + + match target_file { + Ok(target_file) => { + if target_file.items.is_empty() { + log::info!("Creating file in target repo: {}", file_path); + gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| { + LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path)) + })?; + } else { + log::info!("File already exists in target repo: {}", file_path); + } + }, + Err(_) => { + log::info!("Creating file in target repo: {}", file_path); + gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| { + LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path)) + })?; + }, + } + } + } + + Ok(()) } \ No newline at end of file diff --git a/fplus-lib/src/external_services/github.rs b/fplus-lib/src/external_services/github.rs index 58ba291e..e10afa58 100644 --- a/fplus-lib/src/external_services/github.rs +++ b/fplus-lib/src/external_services/github.rs @@ -711,4 +711,23 @@ impl GithubWrapper { .await?; Ok(pull_request.head.ref_field.clone()) } + + pub async fn get_files_from_public_repo( + &self, + owner: &str, + repo: &str, + path: Option<&str> + ) -> Result { + let branch = "main"; + let octocrab = Octocrab::builder().build()?; + let gh = octocrab.repos(owner, repo); + + //if path is not provided, take all files from root + let contents_items = match path { + Some(p) => gh.get_content().r#ref(branch).path(p).send().await?, + None => gh.get_content().r#ref(branch).send().await?, + }; + + Ok(contents_items) + } }