From 6d67a54f41db939b6ccf3cd6c2b1bbcc946d74a7 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 11 Feb 2025 14:45:51 +0530 Subject: [PATCH 01/49] Create Jenkinsfile --- Jenkinsfile | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..030dbbca --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,73 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + + parameters { + string(name: 'AWS_REGION', defaultValue: 'us-east-1', description: 'AWS Region (e.g., us-east-1)') + string(name: 'PROJECT_NAME', defaultValue: 'flotorch', description: 'Project Name (e.g., flotorch)') + string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') + string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') + string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') + string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') + password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') + password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') + string(name: 'TEMPLATE_VERSION', defaultValue: '', description: 'FloTorch Template Version') + } + + stages { + stage('Install dependencies') { + steps { + sh """ + echo "Installing AWS CLI..." + apt install -y unzip curl + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + ./aws/install + aws --version + """ + } + } + + stage('Checkout Repository') { + steps { + script { + git branch: 'main', url: 'https://github.com/FissionAI/FloTorch.git' + } + } + } + + stage('Deploy FloTorch Master Stack') { + steps { + sh """ + echo "Starting FloTorch Stack Deployment..." + aws cloudformation create-stack \ + --region ${AWS_REGION} \ + --stack-name flotorch-stack \ + --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ + --parameters \ + ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ + ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ + ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ + ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ + ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ + ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ + ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} + """ + } + } + } + + post { + success { + echo "FloTorch Stack deployed successfully!" + } + failure { + echo "Deployment failed!" + } + } +} From dc5780987c7f6e53c1a24bab06919b6076251f38 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 11 Feb 2025 14:53:20 +0530 Subject: [PATCH 02/49] Update Jenkinsfile --- Jenkinsfile | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 030dbbca..d87e2488 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'ubuntu:latest' + image 'unbuntu:latest' args '-u root' } } @@ -18,17 +18,29 @@ pipeline { string(name: 'TEMPLATE_VERSION', defaultValue: '', description: 'FloTorch Template Version') } + environment { + AWS_REGION = "${params.AWS_REGION}" + PROJECT_NAME = "${params.PROJECT_NAME}" + TABLE_SUFFIX = "${params.TABLE_SUFFIX}" + CLIENT_NAME = "${params.CLIENT_NAME}" + CREATED_BY = "${params.CREATED_BY}" + OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" + OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" + NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" + TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" + } + stages { stage('Install dependencies') { steps { - sh """ + sh ''' echo "Installing AWS CLI..." - apt install -y unzip curl + yum install -y unzip curl curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install aws --version - """ + ''' } } @@ -42,8 +54,9 @@ pipeline { stage('Deploy FloTorch Master Stack') { steps { - sh """ + sh ''' echo "Starting FloTorch Stack Deployment..." + aws cloudformation create-stack \ --region ${AWS_REGION} \ --stack-name flotorch-stack \ @@ -57,7 +70,9 @@ pipeline { ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} - """ + + echo "CloudFormation Stack Deployment Successful!" + ''' } } } @@ -67,7 +82,7 @@ pipeline { echo "FloTorch Stack deployed successfully!" } failure { - echo "Deployment failed!" + echo "Deployment failed!." } } } From 204f22bf41cb109a1ba508779a8d9be5816f0193 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 11 Feb 2025 15:11:33 +0530 Subject: [PATCH 03/49] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d87e2488..d9f7fb11 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'unbuntu:latest' + image 'ubuntu:latest' args '-u root' } } From 0b8d44523203b3fe610bb86ad911b2bfa158636c Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 10:06:55 +0530 Subject: [PATCH 04/49] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d9f7fb11..f25f0f16 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ pipeline { steps { sh ''' echo "Installing AWS CLI..." - yum install -y unzip curl + apt install -y unzip curl curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install From 14bd123b1d62341a8e207aef80850f0b2e1faf4e Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 10:12:06 +0530 Subject: [PATCH 05/49] Update Jenkinsfile --- Jenkinsfile | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f25f0f16..fd11b512 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,27 +7,11 @@ pipeline { } parameters { - string(name: 'AWS_REGION', defaultValue: 'us-east-1', description: 'AWS Region (e.g., us-east-1)') - string(name: 'PROJECT_NAME', defaultValue: 'flotorch', description: 'Project Name (e.g., flotorch)') - string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') - string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') - string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') - string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') - password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') - password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') - string(name: 'TEMPLATE_VERSION', defaultValue: '', description: 'FloTorch Template Version') + // ... keeping existing parameters ... } environment { - AWS_REGION = "${params.AWS_REGION}" - PROJECT_NAME = "${params.PROJECT_NAME}" - TABLE_SUFFIX = "${params.TABLE_SUFFIX}" - CLIENT_NAME = "${params.CLIENT_NAME}" - CREATED_BY = "${params.CREATED_BY}" - OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" - OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" - NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" - TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" + // ... keeping existing environment variables ... } stages { @@ -35,7 +19,8 @@ pipeline { steps { sh ''' echo "Installing AWS CLI..." - apt install -y unzip curl + apt-get update + apt-get install -y unzip curl curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install @@ -54,9 +39,8 @@ pipeline { stage('Deploy FloTorch Master Stack') { steps { - sh ''' + sh """ echo "Starting FloTorch Stack Deployment..." - aws cloudformation create-stack \ --region ${AWS_REGION} \ --stack-name flotorch-stack \ @@ -70,9 +54,7 @@ pipeline { ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} - - echo "CloudFormation Stack Deployment Successful!" - ''' + """ } } } @@ -82,7 +64,7 @@ pipeline { echo "FloTorch Stack deployed successfully!" } failure { - echo "Deployment failed!." + echo "Deployment failed!" } } } From 1b6c01653b2926d25d00d9cc1be6f13179aba25d Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 10:13:38 +0530 Subject: [PATCH 06/49] Update Jenkinsfile --- Jenkinsfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fd11b512..afd7fe1d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,17 +1,21 @@ pipeline { agent { docker { - image 'ubuntu:latest' + image 'ubuntu:latest' args '-u root' } } parameters { - // ... keeping existing parameters ... - } - - environment { - // ... keeping existing environment variables ... + string(name: 'AWS_REGION', defaultValue: 'us-east-1', description: 'AWS Region (e.g., us-east-1)') + string(name: 'PROJECT_NAME', defaultValue: 'flotorch', description: 'Project Name (e.g., flotorch)') + string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') + string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') + string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') + string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') + password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') + password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') + string(name: 'TEMPLATE_VERSION', defaultValue: '', description: 'FloTorch Template Version') } stages { From a410534b835f2fff587c298d8318a2df9300ac78 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 10:54:52 +0530 Subject: [PATCH 07/49] Update Jenkinsfile --- Jenkinsfile | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index afd7fe1d..a41f93ed 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,24 +41,29 @@ pipeline { } } - stage('Deploy FloTorch Master Stack') { + stage('Deploy FloTorch Master Stack') { steps { - sh """ - echo "Starting FloTorch Stack Deployment..." - aws cloudformation create-stack \ - --region ${AWS_REGION} \ - --stack-name flotorch-stack \ - --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ - --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ - --parameters \ - ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ - ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ - ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ - ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ - ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ - ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ - ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} - """ + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh ''' + echo "Starting FloTorch Stack Deployment..." + + aws cloudformation create-stack \ + --region ${AWS_REGION} \ + --stack-name flotorch-stack \ + --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ + --parameters \ + ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ + ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ + ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ + ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ + ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ + ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ + ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} + + echo "CloudFormation Stack Deployment Successful!" + ''' + } } } } @@ -71,4 +76,13 @@ pipeline { echo "Deployment failed!" } } + + post { + success { + echo "FloTorch Stack deployed successfully!" + } + failure { + echo "Deployment failed!" + } + } } From 8a3124e47acaceb5cf7d65c1297a76bbdd1e5ee1 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 10:58:06 +0530 Subject: [PATCH 08/49] Update Jenkinsfile --- Jenkinsfile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a41f93ed..9b08ab42 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,13 +76,5 @@ pipeline { echo "Deployment failed!" } } - - post { - success { - echo "FloTorch Stack deployed successfully!" - } - failure { - echo "Deployment failed!" - } - } } + From a34daf8d9841f4bfbd8946efeba3400211ca4586 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 11:07:24 +0530 Subject: [PATCH 09/49] Update Jenkinsfile --- Jenkinsfile | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9b08ab42..1ff55c92 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,15 +21,28 @@ pipeline { stages { stage('Install dependencies') { steps { - sh ''' - echo "Installing AWS CLI..." - apt-get update - apt-get install -y unzip curl - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - ./aws/install - aws --version - ''' + script { + sh ''' + echo "Installing AWS CLI..." + apt-get update + apt-get install -y unzip curl + + # Check if AWS CLI is already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + ./aws/install + fi + + # Verify AWS CLI installation + aws --version || { + echo "AWS CLI installation failed" + exit 1 + } + echo "AWS CLI installation successful" + ''' + } } } From 3c19b768308e1f30e00e20fa3218f862acbb4004 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 11:10:44 +0530 Subject: [PATCH 10/49] Update Jenkinsfile --- Jenkinsfile | 55 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1ff55c92..be77cfe1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'ubuntu:latest' + image 'ubuntu:latest' args '-u root' } } @@ -18,6 +18,18 @@ pipeline { string(name: 'TEMPLATE_VERSION', defaultValue: '', description: 'FloTorch Template Version') } + environment { + AWS_REGION = "${params.AWS_REGION}" + PROJECT_NAME = "${params.PROJECT_NAME}" + TABLE_SUFFIX = "${params.TABLE_SUFFIX}" + CLIENT_NAME = "${params.CLIENT_NAME}" + CREATED_BY = "${params.CREATED_BY}" + OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" + OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" + NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" + TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" + } + stages { stage('Install dependencies') { steps { @@ -26,7 +38,7 @@ pipeline { echo "Installing AWS CLI..." apt-get update apt-get install -y unzip curl - + # Check if AWS CLI is already installed if ! command -v aws &> /dev/null; then echo "AWS CLI not found. Installing..." @@ -34,7 +46,7 @@ pipeline { unzip -q awscliv2.zip ./aws/install fi - + # Verify AWS CLI installation aws --version || { echo "AWS CLI installation failed" @@ -54,27 +66,31 @@ pipeline { } } - stage('Deploy FloTorch Master Stack') { + stage('Deploy FloTorch Master Stack') { steps { withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { sh ''' - echo "Starting FloTorch Stack Deployment..." + echo "Starting FloTorch Stack Deployment..." + aws cloudformation create-stack \ + --region ${AWS_REGION} \ + --stack-name flotorch-stack \ + --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ + --parameters \ + ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ + ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ + ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ + ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ + ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ + ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ + ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} - aws cloudformation create-stack \ - --region ${AWS_REGION} \ - --stack-name flotorch-stack \ - --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ - --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ - --parameters \ - ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ - ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ - ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ - ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ - ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ - ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ - ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} + echo "Waiting for stack creation to complete..." + aws cloudformation wait stack-create-complete \ + --stack-name flotorch-stack \ + --region ${AWS_REGION} - echo "CloudFormation Stack Deployment Successful!" + echo "CloudFormation Stack Deployment Successful!" ''' } } @@ -90,4 +106,3 @@ pipeline { } } } - From 81ca9e87eab9f2933fd9e0b0b833fcd28c13b205 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 11:14:33 +0530 Subject: [PATCH 11/49] Update Jenkinsfile --- Jenkinsfile | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index be77cfe1..8f823e1e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,31 +32,35 @@ pipeline { stages { stage('Install dependencies') { - steps { - script { - sh ''' - echo "Installing AWS CLI..." - apt-get update - apt-get install -y unzip curl - - # Check if AWS CLI is already installed - if ! command -v aws &> /dev/null; then - echo "AWS CLI not found. Installing..." - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -q awscliv2.zip - ./aws/install - fi - - # Verify AWS CLI installation - aws --version || { - echo "AWS CLI installation failed" - exit 1 - } - echo "AWS CLI installation successful" - ''' + steps { + script { + sh ''' + echo "Installing AWS CLI..." + apt-get update + apt-get install -y unzip curl + + # Check if AWS CLI is already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + # Force overwrite during unzip + unzip -o -q awscliv2.zip + ./aws/install --update + else + echo "AWS CLI is already installed" + aws --version + fi + + # Verify AWS CLI installation + aws --version || { + echo "AWS CLI installation failed" + exit 1 } - } + echo "AWS CLI installation successful" + ''' } + } +} stage('Checkout Repository') { steps { From 8fa9af9ece8887793d06f0b7ee98d5ad175e7a3d Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 12:20:59 +0530 Subject: [PATCH 12/49] Update Jenkinsfile --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8f823e1e..4b5bf098 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,7 +15,6 @@ pipeline { string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') - string(name: 'TEMPLATE_VERSION', defaultValue: '', description: 'FloTorch Template Version') } environment { @@ -27,7 +26,6 @@ pipeline { OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" - TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" } stages { @@ -78,7 +76,7 @@ pipeline { aws cloudformation create-stack \ --region ${AWS_REGION} \ --stack-name flotorch-stack \ - --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ + --template-url https://flotorch-public.s3.amazonaws.com/templates/master-template.yaml \ --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ --parameters \ ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ From 0d960500788d765407d80987dca9551d3b574e91 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 12:29:03 +0530 Subject: [PATCH 13/49] Update Jenkinsfile --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 4b5bf098..3e564b16 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,6 +86,7 @@ pipeline { ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} + ParameterKey=PrerequisitesMet,ParameterValue=true echo "Waiting for stack creation to complete..." aws cloudformation wait stack-create-complete \ From 5a89553e3f23ee58ebd9f039707aa6fe12a5bfa9 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 12:32:33 +0530 Subject: [PATCH 14/49] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3e564b16..5c44b81f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ - ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} + ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} \ ParameterKey=PrerequisitesMet,ParameterValue=true echo "Waiting for stack creation to complete..." From 37edd0b93fa7921b012a4ec29c23ca6084f65a4f Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 12 Feb 2025 12:35:45 +0530 Subject: [PATCH 15/49] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5c44b81f..f41526d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ pipeline { ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} \ - ParameterKey=PrerequisitesMet,ParameterValue=true + ParameterKey=PrerequisitesMet,ParameterValue=yes echo "Waiting for stack creation to complete..." aws cloudformation wait stack-create-complete \ From b8ccbef8307ab3ce50a4897b0edf4fc34e22f1ac Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Thu, 13 Feb 2025 11:47:52 +0530 Subject: [PATCH 16/49] Update Jenkinsfile --- Jenkinsfile | 59 ++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f41526d3..53d35cfa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,6 +15,7 @@ pipeline { string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') + string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1') } environment { @@ -26,39 +27,39 @@ pipeline { OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" + TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" } stages { stage('Install dependencies') { - steps { - script { - sh ''' - echo "Installing AWS CLI..." - apt-get update - apt-get install -y unzip curl - - # Check if AWS CLI is already installed - if ! command -v aws &> /dev/null; then - echo "AWS CLI not found. Installing..." - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - # Force overwrite during unzip - unzip -o -q awscliv2.zip - ./aws/install --update - else - echo "AWS CLI is already installed" - aws --version - fi - - # Verify AWS CLI installation - aws --version || { - echo "AWS CLI installation failed" - exit 1 + steps { + script { + sh ''' + echo "Installing AWS CLI..." + apt-get update + apt-get install -y unzip curl + + # Check if AWS CLI is already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + else + echo "AWS CLI is already installed" + aws --version + fi + + # Verify AWS CLI installation + aws --version || { + echo "AWS CLI installation failed" + exit 1 + } + echo "AWS CLI installation successful" + ''' } - echo "AWS CLI installation successful" - ''' + } } - } -} stage('Checkout Repository') { steps { @@ -73,16 +74,18 @@ pipeline { withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { sh ''' echo "Starting FloTorch Stack Deployment..." + aws cloudformation create-stack \ --region ${AWS_REGION} \ --stack-name flotorch-stack \ - --template-url https://flotorch-public.s3.amazonaws.com/templates/master-template.yaml \ + --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ --parameters \ ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ + ParameterKey=TemplateVersion,ParameterValue=${TEMPLATE_VERSION} \ ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} \ From 1794554b79a06cda92dfc94073a87e482bf40815 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Thu, 13 Feb 2025 11:50:41 +0530 Subject: [PATCH 17/49] Update Jenkinsfile --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 53d35cfa..cc8599d0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,10 +12,10 @@ pipeline { string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') + string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1') string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') - string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1') } environment { @@ -24,10 +24,10 @@ pipeline { TABLE_SUFFIX = "${params.TABLE_SUFFIX}" CLIENT_NAME = "${params.CLIENT_NAME}" CREATED_BY = "${params.CREATED_BY}" + TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" - TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" } stages { @@ -85,7 +85,7 @@ pipeline { ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ - ParameterKey=TemplateVersion,ParameterValue=${TEMPLATE_VERSION} \ + ParameterKey=TemplateVersion,ParameterValue=${TEMPLATE_VERSION} \ ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} \ From ff2a69679801a861fac855a3c57135845db6f94e Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Thu, 13 Feb 2025 11:53:59 +0530 Subject: [PATCH 18/49] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index cc8599d0..2e752e98 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ pipeline { string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') - string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1') + string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1)') string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') From 365463fdeb054a424de785f6ca9511a0ad8723c1 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Thu, 13 Feb 2025 18:06:35 +0530 Subject: [PATCH 19/49] Create Jenkinsfile --- Jenkins/Jenkinsfile | 114 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 Jenkins/Jenkinsfile diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile new file mode 100644 index 00000000..2e752e98 --- /dev/null +++ b/Jenkins/Jenkinsfile @@ -0,0 +1,114 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + + parameters { + string(name: 'AWS_REGION', defaultValue: 'us-east-1', description: 'AWS Region (e.g., us-east-1)') + string(name: 'PROJECT_NAME', defaultValue: 'flotorch', description: 'Project Name (e.g., flotorch)') + string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') + string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') + string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') + string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1)') + string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') + password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') + password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') + } + + environment { + AWS_REGION = "${params.AWS_REGION}" + PROJECT_NAME = "${params.PROJECT_NAME}" + TABLE_SUFFIX = "${params.TABLE_SUFFIX}" + CLIENT_NAME = "${params.CLIENT_NAME}" + CREATED_BY = "${params.CREATED_BY}" + TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" + OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" + OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" + NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" + } + + stages { + stage('Install dependencies') { + steps { + script { + sh ''' + echo "Installing AWS CLI..." + apt-get update + apt-get install -y unzip curl + + # Check if AWS CLI is already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + else + echo "AWS CLI is already installed" + aws --version + fi + + # Verify AWS CLI installation + aws --version || { + echo "AWS CLI installation failed" + exit 1 + } + echo "AWS CLI installation successful" + ''' + } + } + } + + stage('Checkout Repository') { + steps { + script { + git branch: 'main', url: 'https://github.com/FissionAI/FloTorch.git' + } + } + } + + stage('Deploy FloTorch Master Stack') { + steps { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh ''' + echo "Starting FloTorch Stack Deployment..." + + aws cloudformation create-stack \ + --region ${AWS_REGION} \ + --stack-name flotorch-stack \ + --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ + --parameters \ + ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ + ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ + ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ + ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ + ParameterKey=TemplateVersion,ParameterValue=${TEMPLATE_VERSION} \ + ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ + ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ + ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} \ + ParameterKey=PrerequisitesMet,ParameterValue=yes + + echo "Waiting for stack creation to complete..." + aws cloudformation wait stack-create-complete \ + --stack-name flotorch-stack \ + --region ${AWS_REGION} + + echo "CloudFormation Stack Deployment Successful!" + ''' + } + } + } + } + + post { + success { + echo "FloTorch Stack deployed successfully!" + } + failure { + echo "Deployment failed!" + } + } +} From 1fb55edac019ad201989184bba67a27a0431db21 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Thu, 13 Feb 2025 18:07:41 +0530 Subject: [PATCH 20/49] Delete Dockerfile --- Dockerfile | 126 ----------------------------------------------------- 1 file changed, 126 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 01cf5083..00000000 --- a/Dockerfile +++ /dev/null @@ -1,126 +0,0 @@ -# Use Amazon Linux 2023 as base image -FROM amazonlinux:2023 - -# Install system dependencies in stages to handle package conflicts properly -RUN dnf update -y && \ - dnf clean all - -# Install basic build tools first -RUN dnf install -y \ - gcc-c++ \ - make \ - shadow-utils \ - openssl \ - openssl-devel \ - && dnf clean all - -# Install Python and development tools -RUN dnf install -y \ - python3 \ - python3-pip \ - git \ - && dnf clean all - -# Install compression tools -RUN dnf install -y \ - tar \ - gzip \ - unzip \ - && dnf clean all - -# Install jq -RUN dnf install -y \ - jq && \ - dnf clean all - -# Install Docker and required dependencies -RUN dnf install -y docker iptables procps && \ - dnf clean all - -# Create necessary directories with proper permissions -RUN mkdir -p /run/docker && \ - mkdir -p /var/lib/docker && \ - mkdir -p /etc/docker - -# Configure Docker daemon -RUN echo '{"storage-driver": "vfs"}' > /etc/docker/daemon.json - -# Install Node.js and npm using nvm -ENV NVM_DIR=/root/.nvm -ENV NODE_VERSION=22.12.0 - -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \ - . $NVM_DIR/nvm.sh && \ - nvm install $NODE_VERSION && \ - nvm alias default $NODE_VERSION && \ - nvm use default - -# Add node and npm to path so the commands are available -ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH - -# Verify Node.js and npm versions -RUN node -v && npm -v - -# Install AWS CLI v2 -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && \ - ./aws/install && \ - rm -rf aws awscliv2.zip - -# Install AWS CDK globally -RUN npm install -g aws-cdk && \ - cdk --version - -# Set working directory -WORKDIR /FloTorch - -# Copy the entire FloTorch directory -COPY . . - -# Make deploy.sh executable -RUN chmod +x /FloTorch/cdk/deploy.sh - -# Install required Python packages -RUN pip3 install -r cdk/requirements.txt --upgrade - -# Create the entrypoint script -RUN echo '#!/bin/bash' > /entrypoint.sh && \ - echo 'set -e' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Start Docker daemon if not running' >> /entrypoint.sh && \ - echo 'if ! pgrep dockerd >/dev/null 2>&1; then' >> /entrypoint.sh && \ - echo ' echo "Starting Docker daemon..."' >> /entrypoint.sh && \ - echo ' dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --storage-driver=vfs &' >> /entrypoint.sh && \ - echo ' echo "Waiting for Docker daemon to start..."' >> /entrypoint.sh && \ - echo ' for i in $(seq 1 30); do' >> /entrypoint.sh && \ - echo ' if docker info >/dev/null 2>&1; then' >> /entrypoint.sh && \ - echo ' echo "Docker daemon started successfully"' >> /entrypoint.sh && \ - echo ' break' >> /entrypoint.sh && \ - echo ' fi' >> /entrypoint.sh && \ - echo ' echo "Waiting for Docker daemon... ($i/30)"' >> /entrypoint.sh && \ - echo ' sleep 1' >> /entrypoint.sh && \ - echo ' done' >> /entrypoint.sh && \ - echo 'fi' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Print environment for debugging' >> /entrypoint.sh && \ - echo 'echo "AWS credentials setup..."' >> /entrypoint.sh && \ - echo 'aws configure list' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Verify required environment variables' >> /entrypoint.sh && \ - echo 'if [ -z "${AWS_ACCOUNT_ID}" ] || [ -z "${AWS_DEFAULT_REGION}" ]; then' >> /entrypoint.sh && \ - echo ' echo "Error: AWS_ACCOUNT_ID and AWS_DEFAULT_REGION must be provided"' >> /entrypoint.sh && \ - echo ' exit 1' >> /entrypoint.sh && \ - echo 'fi' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Run deployment with verbose output' >> /entrypoint.sh && \ - echo 'cd /FloTorch/cdk' >> /entrypoint.sh && \ - echo 'echo "Starting deployment process..."' >> /entrypoint.sh && \ - echo 'echo "Using AWS Account: ${AWS_ACCOUNT_ID}"' >> /entrypoint.sh && \ - echo 'echo "Using AWS Region: ${AWS_DEFAULT_REGION}"' >> /entrypoint.sh && \ - echo './deploy.sh' >> /entrypoint.sh && \ - echo 'echo "Deployment completed."' >> /entrypoint.sh && \ - chmod +x /entrypoint.sh - -# Set the entrypoint -ENTRYPOINT ["/entrypoint.sh"] From 1eb0e859f283e121613352169856e8433f458211 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 12:34:57 +0530 Subject: [PATCH 21/49] Create Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 136 +++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 Jenkins/Jenkinsfile.deployment diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment new file mode 100644 index 00000000..ed42236c --- /dev/null +++ b/Jenkins/Jenkinsfile.deployment @@ -0,0 +1,136 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + + parameters { + // Interactive Parameters + choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') + string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') + + // Pre-configured Environment Values (non-interactive) + // Dev Environment (fixed values) + string(name: 'DEV_ENV', defaultValue: 'dampen') + string(name: 'DEV_URL', defaultValue: 'https://radd4urst2.us-east-1.awsapprunner.com') + string(name: 'DEV_USERNAME', defaultValue: 'admin') + password(name: 'DEV_PASSWORD', defaultValue: 'jkhg78BKUYBKGUY7*nhiH') + + // QA Environment (fixed values) + string(name: 'QA_ENV', defaultValue: 'iamdqa') + string(name: 'QA_URL', defaultValue: 'https://tmmqcidccg.us-east-1.awsapprunner.com') + string(name: 'QA_USERNAME', defaultValue: 'admin') + password(name: 'QA_PASSWORD', defaultValue: 'jkhrg&*ObliubUH*(8') + + // Prod Environment (fixed values) + string(name: 'PROD_USERNAME', defaultValue: 'admin') + password(name: 'PROD_PASSWORD', defaultValue: '') + } + + environment { + ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' + AWS_REGION = 'us-east-1' + } + + stages { + stage('Install Dependencies') { + steps { + script { + sh ''' + apt-get update + apt-get install -y docker.io curl unzip git + + # Install AWS CLI + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + ''' + } + } + } + + stage('Checkout') { + steps { + // Clean workspace before checkout + cleanWs() + // Checkout source code + git branch: 'main', + url: 'https://github.com/FissionAI/FloTorch.git' + } + } + + stage('Parameter Validation') { + steps { + script { + echo "Selected Environment: ${params.ENVIRONMENT}" + echo "Build Version: ${params.BUILD_VERSION}" + echo "Skip Tests: ${params.SKIP_TESTS}" + + // Additional validation for Production + if (params.ENVIRONMENT == 'prod') { + input message: 'Confirm Production Deployment', ok: 'Proceed' + } + } + } + } + + + stage('Deploy to Environment') { + steps { + script { + def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : + params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' + + echo "Deploying to ${params.ENVIRONMENT} environment: ${envName}" + echo "Using build version: ${params.BUILD_VERSION}" + + // AWS ECR Login + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ + + // Build and push Docker images + def services = [ + [name: 'app', dockerfile: 'app/Dockerfile'], + [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], + [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], + [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], + [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] + ] + + services.each { service -> + sh """ + docker build -t ${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION} \ + -f ${service.dockerfile} --push . + """ + } + + dir('lambda_handlers') { + sh """ + docker build -t ${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION} \ + -f cost_handler/Dockerfile --push . + """ + } + } + } + } + } + } + + post { + success { + echo "Deployment successful to ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" + } + failure { + echo "Deployment failed for ${params.ENVIRONMENT}" + } + always { + cleanWs() + sh 'docker system prune -f' + } + } +} From 2278011b97e0ac302857497c0376b2b71ced42ef Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 13:12:02 +0530 Subject: [PATCH 22/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 80 ++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index ed42236c..643379af 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -75,52 +75,56 @@ pipeline { } } } + stage('Deploy to Environment') { + steps { + script { + def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : + params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' + + echo "Deploying to ${params.ENVIRONMENT} environment: ${envName}" + echo "Using build version: ${params.BUILD_VERSION}" + + // AWS ECR Login + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ + // Build and push Docker images + def services = [ + [name: 'app', dockerfile: 'app/Dockerfile'], + [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], + [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], + [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], + [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] + ] - stage('Deploy to Environment') { - steps { - script { - def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : - params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' - - echo "Deploying to ${params.ENVIRONMENT} environment: ${envName}" - echo "Using build version: ${params.BUILD_VERSION}" - - // AWS ECR Login - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - sh """ - aws ecr get-login-password --region ${AWS_REGION} | \ - docker login --username AWS --password-stdin ${ECR_REGISTRY} - """ - - // Build and push Docker images - def services = [ - [name: 'app', dockerfile: 'app/Dockerfile'], - [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], - [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], - [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], - [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] - ] - - services.each { service -> - sh """ - docker build -t ${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION} \ - -f ${service.dockerfile} --push . - """ - } + services.each { service -> + def imageTag = "${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION}" + sh """ + # Build image + docker build -t ${imageTag} -f ${service.dockerfile} . + # Push image + docker push ${imageTag} + """ + } - dir('lambda_handlers') { - sh """ - docker build -t ${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION} \ - -f cost_handler/Dockerfile --push . - """ - } - } + dir('lambda_handlers') { + def costComputeTag = "${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION}" + sh """ + # Build cost compute handler + docker build -t ${costComputeTag} -f cost_handler/Dockerfile . + # Push image + docker push ${costComputeTag} + """ } } } } +} + post { success { echo "Deployment successful to ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" From c14db4d76b51a28f5784a50198c887398953ab4f Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 13:18:55 +0530 Subject: [PATCH 23/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 91 ++++++++++++++++------------------ 1 file changed, 42 insertions(+), 49 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 643379af..894d0c27 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -53,9 +53,7 @@ pipeline { stage('Checkout') { steps { - // Clean workspace before checkout cleanWs() - // Checkout source code git branch: 'main', url: 'https://github.com/FissionAI/FloTorch.git' } @@ -66,65 +64,60 @@ pipeline { script { echo "Selected Environment: ${params.ENVIRONMENT}" echo "Build Version: ${params.BUILD_VERSION}" - echo "Skip Tests: ${params.SKIP_TESTS}" - - // Additional validation for Production + if (params.ENVIRONMENT == 'prod') { input message: 'Confirm Production Deployment', ok: 'Proceed' } } } } - stage('Deploy to Environment') { - steps { - script { - def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : - params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' - - echo "Deploying to ${params.ENVIRONMENT} environment: ${envName}" - echo "Using build version: ${params.BUILD_VERSION}" - - // AWS ECR Login - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - sh """ - aws ecr get-login-password --region ${AWS_REGION} | \ - docker login --username AWS --password-stdin ${ECR_REGISTRY} - """ - // Build and push Docker images - def services = [ - [name: 'app', dockerfile: 'app/Dockerfile'], - [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], - [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], - [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], - [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] - ] + stage('Deploy to Environment') { + steps { + script { + def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : + params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' + + echo "Deploying to ${params.ENVIRONMENT} environment: ${envName}" + echo "Using build version: ${params.BUILD_VERSION}" + + // AWS ECR Login + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ - services.each { service -> - def imageTag = "${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION}" - sh """ - # Build image - docker build -t ${imageTag} -f ${service.dockerfile} . - # Push image - docker push ${imageTag} - """ - } + // Build and push Docker images + def services = [ + [name: 'app', dockerfile: 'app/Dockerfile'], + [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], + [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], + [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], + [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] + ] - dir('lambda_handlers') { - def costComputeTag = "${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION}" - sh """ - # Build cost compute handler - docker build -t ${costComputeTag} -f cost_handler/Dockerfile . - # Push image - docker push ${costComputeTag} - """ + services.each { service -> + def imageTag = "${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION}" + sh """ + docker build -t ${imageTag} -f ${service.dockerfile} . + docker push ${imageTag} + """ + } + + dir('lambda_handlers') { + def costComputeTag = "${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION}" + sh """ + docker build -t ${costComputeTag} -f cost_handler/Dockerfile . + docker push ${costComputeTag} + """ + } + } } } } - } -} + } - post { success { echo "Deployment successful to ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" @@ -137,4 +130,4 @@ pipeline { sh 'docker system prune -f' } } -} +} From 4c7d98076b6a3473f3614018181e62a5607bd994 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 16:20:36 +0530 Subject: [PATCH 24/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 108 +++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 12 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 894d0c27..1358ed79 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -41,8 +41,6 @@ pipeline { sh ''' apt-get update apt-get install -y docker.io curl unzip git - - # Install AWS CLI curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip -o -q awscliv2.zip ./aws/install --update @@ -64,7 +62,7 @@ pipeline { script { echo "Selected Environment: ${params.ENVIRONMENT}" echo "Build Version: ${params.BUILD_VERSION}" - + if (params.ENVIRONMENT == 'prod') { input message: 'Confirm Production Deployment', ok: 'Proceed' } @@ -72,17 +70,55 @@ pipeline { } } + stage('Create Deployment Files') { + steps { + script { + // Create entrypoint.sh + writeFile file: 'entrypoint.sh', text: '''#!/bin/bash + htpasswd -bc /etc/nginx/.htpasswd admin ${NGINX_AUTH_PASSWORD} + supervisord -n -c /etc/supervisor/supervisord.conf + ''' + + sh 'chmod +x entrypoint.sh' + + // Create supervisor.conf + writeFile file: 'supervisor.conf', text: '''[supervisord] + nodaemon=true + + [program:nginx] + command=/usr/sbin/nginx -g "daemon off;" + autostart=true + autorestart=true + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/stderr + stderr_logfile_maxbytes=0 + + [program:app] + command=python main.py + directory=/app + autostart=true + autorestart=true + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/stderr + stderr_logfile_maxbytes=0 + ''' + } + } + } + stage('Deploy to Environment') { steps { script { - def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : + def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' - + echo "Deploying to ${params.ENVIRONMENT} environment: ${envName}" echo "Using build version: ${params.BUILD_VERSION}" - - // AWS ECR Login + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + // ECR Login sh """ aws ecr get-login-password --region ${AWS_REGION} | \ docker login --username AWS --password-stdin ${ECR_REGISTRY} @@ -98,17 +134,21 @@ pipeline { ] services.each { service -> - def imageTag = "${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION}" sh """ - docker build -t ${imageTag} -f ${service.dockerfile} . - docker push ${imageTag} + # Build image + docker build -t ${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION} \ + -f ${service.dockerfile} . + # Push image + docker push ${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION} """ } dir('lambda_handlers') { def costComputeTag = "${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION}" sh """ + # Build cost compute handler docker build -t ${costComputeTag} -f cost_handler/Dockerfile . + # Push image docker push ${costComputeTag} """ } @@ -116,7 +156,50 @@ pipeline { } } } - } + + stage('Update Services') { + steps { + script { + def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : + params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' + + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + // Update App Runner Service + sh """ + aws apprunner update-service \ + --service-name flotorch-app-${envName} \ + --source-configuration imageRepository={imageIdentifier=${ECR_REGISTRY}/flotorch-app-${envName}:${params.BUILD_VERSION}} + """ + + // Update ECS Services + sh """ + aws ecs update-service \ + --cluster flotorch-${envName} \ + --service flotorch-indexing-${envName} \ + --force-new-deployment + + aws ecs update-service \ + --cluster flotorch-${envName} \ + --service flotorch-retriever-${envName} \ + --force-new-deployment + + aws ecs update-service \ + --cluster flotorch-${envName} \ + --service flotorch-evaluation-${envName} \ + --force-new-deployment + """ + + // Update Lambda Function + sh """ + aws lambda update-function-code \ + --function-name flotorch-costcompute-${envName} \ + --image-uri ${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION} + """ + } + } + } + } + } post { success { @@ -130,4 +213,5 @@ pipeline { sh 'docker system prune -f' } } -} +} + From 0d9f98cffab12542bf0078bd67fd8dc08968937b Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 16:27:23 +0530 Subject: [PATCH 25/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 158 +++------------------------------ 1 file changed, 10 insertions(+), 148 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 1358ed79..63556a9a 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -7,31 +7,14 @@ pipeline { } parameters { - // Interactive Parameters choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') - - // Pre-configured Environment Values (non-interactive) - // Dev Environment (fixed values) - string(name: 'DEV_ENV', defaultValue: 'dampen') - string(name: 'DEV_URL', defaultValue: 'https://radd4urst2.us-east-1.awsapprunner.com') - string(name: 'DEV_USERNAME', defaultValue: 'admin') - password(name: 'DEV_PASSWORD', defaultValue: 'jkhg78BKUYBKGUY7*nhiH') - - // QA Environment (fixed values) - string(name: 'QA_ENV', defaultValue: 'iamdqa') - string(name: 'QA_URL', defaultValue: 'https://tmmqcidccg.us-east-1.awsapprunner.com') - string(name: 'QA_USERNAME', defaultValue: 'admin') - password(name: 'QA_PASSWORD', defaultValue: 'jkhrg&*ObliubUH*(8') - - // Prod Environment (fixed values) - string(name: 'PROD_USERNAME', defaultValue: 'admin') - password(name: 'PROD_PASSWORD', defaultValue: '') } environment { ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' AWS_REGION = 'us-east-1' + ENV_NAME = "${params.ENVIRONMENT == 'dev' ? 'dampen' : params.ENVIRONMENT == 'qa' ? 'iamdqa' : 'prod'}" } stages { @@ -57,66 +40,9 @@ pipeline { } } - stage('Parameter Validation') { - steps { - script { - echo "Selected Environment: ${params.ENVIRONMENT}" - echo "Build Version: ${params.BUILD_VERSION}" - - if (params.ENVIRONMENT == 'prod') { - input message: 'Confirm Production Deployment', ok: 'Proceed' - } - } - } - } - - stage('Create Deployment Files') { - steps { - script { - // Create entrypoint.sh - writeFile file: 'entrypoint.sh', text: '''#!/bin/bash - htpasswd -bc /etc/nginx/.htpasswd admin ${NGINX_AUTH_PASSWORD} - supervisord -n -c /etc/supervisor/supervisord.conf - ''' - - sh 'chmod +x entrypoint.sh' - - // Create supervisor.conf - writeFile file: 'supervisor.conf', text: '''[supervisord] - nodaemon=true - - [program:nginx] - command=/usr/sbin/nginx -g "daemon off;" - autostart=true - autorestart=true - stdout_logfile=/dev/stdout - stdout_logfile_maxbytes=0 - stderr_logfile=/dev/stderr - stderr_logfile_maxbytes=0 - - [program:app] - command=python main.py - directory=/app - autostart=true - autorestart=true - stdout_logfile=/dev/stdout - stdout_logfile_maxbytes=0 - stderr_logfile=/dev/stderr - stderr_logfile_maxbytes=0 - ''' - } - } - } - - stage('Deploy to Environment') { + stage('Build and Push Images') { steps { script { - def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : - params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' - - echo "Deploying to ${params.ENVIRONMENT} environment: ${envName}" - echo "Using build version: ${params.BUILD_VERSION}" - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { // ECR Login sh """ @@ -124,76 +50,13 @@ pipeline { docker login --username AWS --password-stdin ${ECR_REGISTRY} """ - // Build and push Docker images - def services = [ - [name: 'app', dockerfile: 'app/Dockerfile'], - [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], - [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], - [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], - [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] - ] - - services.each { service -> - sh """ - # Build image - docker build -t ${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION} \ - -f ${service.dockerfile} . - # Push image - docker push ${ECR_REGISTRY}/flotorch-${service.name}-${envName}:${params.BUILD_VERSION} - """ - } - - dir('lambda_handlers') { - def costComputeTag = "${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION}" - sh """ - # Build cost compute handler - docker build -t ${costComputeTag} -f cost_handler/Dockerfile . - # Push image - docker push ${costComputeTag} - """ - } - } - } - } - } - - stage('Update Services') { - steps { - script { - def envName = params.ENVIRONMENT == 'dev' ? params.DEV_ENV : - params.ENVIRONMENT == 'qa' ? params.QA_ENV : 'prod' - - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - // Update App Runner Service - sh """ - aws apprunner update-service \ - --service-name flotorch-app-${envName} \ - --source-configuration imageRepository={imageIdentifier=${ECR_REGISTRY}/flotorch-app-${envName}:${params.BUILD_VERSION}} - """ - - // Update ECS Services - sh """ - aws ecs update-service \ - --cluster flotorch-${envName} \ - --service flotorch-indexing-${envName} \ - --force-new-deployment - - aws ecs update-service \ - --cluster flotorch-${envName} \ - --service flotorch-retriever-${envName} \ - --force-new-deployment - - aws ecs update-service \ - --cluster flotorch-${envName} \ - --service flotorch-evaluation-${envName} \ - --force-new-deployment - """ - - // Update Lambda Function + // Build and push app service + def appImageTag = "${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION}" sh """ - aws lambda update-function-code \ - --function-name flotorch-costcompute-${envName} \ - --image-uri ${ECR_REGISTRY}/flotorch-costcompute-${envName}:${params.BUILD_VERSION} + # Build image + docker build -t ${appImageTag} -f app/Dockerfile . + # Push image + docker push ${appImageTag} """ } } @@ -203,10 +66,10 @@ pipeline { post { success { - echo "Deployment successful to ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" + echo "Successfully built and pushed images for ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" } failure { - echo "Deployment failed for ${params.ENVIRONMENT}" + echo "Failed to build/push images for ${params.ENVIRONMENT}" } always { cleanWs() @@ -214,4 +77,3 @@ pipeline { } } } - From 93ad533b5831c9272e37fc82d99df600dc8e8bb3 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 17:02:47 +0530 Subject: [PATCH 26/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 63556a9a..2ab6b54b 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -36,33 +36,37 @@ pipeline { steps { cleanWs() git branch: 'main', - url: 'https://github.com/FissionAI/FloTorch.git' + url: 'https://github.com/shivani-fission/FloTorch.git' } } - stage('Build and Push Images') { - steps { - script { - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - // ECR Login - sh """ - aws ecr get-login-password --region ${AWS_REGION} | \ - docker login --username AWS --password-stdin ${ECR_REGISTRY} - """ - - // Build and push app service - def appImageTag = "${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION}" - sh """ - # Build image - docker build -t ${appImageTag} -f app/Dockerfile . - # Push image - docker push ${appImageTag} - """ - } +stage('Build and Push Images') { + steps { + script { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + try { + // ECR Login + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ + + // Build and push app service + def appImageTag = "${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION}" + sh """ + # Build image + docker build -t ${appImageTag} -f app/Dockerfile . || exit 1 + # Push image + docker push ${appImageTag} || exit 1 + """ + } catch (Exception e) { + echo "Error during build/push: ${e.getMessage()}" + throw e } } } } +} post { success { From 89de9890bd480c50b47692bb9e735771c5f39605 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 17:03:59 +0530 Subject: [PATCH 27/49] Create entrypoint.sh --- app/entrypoint.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 app/entrypoint.sh diff --git a/app/entrypoint.sh b/app/entrypoint.sh new file mode 100644 index 00000000..366ff887 --- /dev/null +++ b/app/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Create nginx directory if it doesn't exist +mkdir -p /etc/nginx +NGINX_USER="${NGINX_AUTH_USER:-admin}" +NGINX_PASS="${NGINX_AUTH_PASSWORD:-Flotorch@123}" +if [ ! -f /etc/nginx/.htpasswd ] || [ ! -s /etc/nginx/.htpasswd ]; then + echo "Creating new .htpasswd file with provided credentials" + htpasswd -cb /etc/nginx/.htpasswd "$NGINX_USER" "$NGINX_PASS" +else + echo "Updating existing .htpasswd file with new credentials" + htpasswd -b /etc/nginx/.htpasswd "$NGINX_USER" "$NGINX_PASS" +fi +exec supervisord -n From 8f2424dddd2cb9b368146168bd6babd464e95774 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 17:04:51 +0530 Subject: [PATCH 28/49] Create flotorch.conf --- app/supervisor/flotorch.conf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 app/supervisor/flotorch.conf diff --git a/app/supervisor/flotorch.conf b/app/supervisor/flotorch.conf new file mode 100644 index 00000000..8b0e8300 --- /dev/null +++ b/app/supervisor/flotorch.conf @@ -0,0 +1,21 @@ +[supervisord] +nodaemon=true + +[program:nginx] +command=nginx -g "daemon off;" +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:uvicorn] +command=uvicorn app.main:app --host 0.0.0.0 --port 8000 +directory=/app +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 From 26aeb8b9ae87a497046139761e5d539fdf08042e Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 17:09:22 +0530 Subject: [PATCH 29/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 2ab6b54b..5de6e7cf 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -66,6 +66,7 @@ stage('Build and Push Images') { } } } +} } post { From 663b986d411cccb659fcf65b168cb32cd0da2f90 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 17:10:40 +0530 Subject: [PATCH 30/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 50 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 5de6e7cf..633b36ed 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -40,34 +40,34 @@ pipeline { } } -stage('Build and Push Images') { - steps { - script { - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - try { - // ECR Login - sh """ - aws ecr get-login-password --region ${AWS_REGION} | \ - docker login --username AWS --password-stdin ${ECR_REGISTRY} - """ - - // Build and push app service - def appImageTag = "${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION}" - sh """ - # Build image - docker build -t ${appImageTag} -f app/Dockerfile . || exit 1 - # Push image - docker push ${appImageTag} || exit 1 - """ - } catch (Exception e) { - echo "Error during build/push: ${e.getMessage()}" - throw e + stage('Build and Push Images') { + steps { + script { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + try { + // ECR Login + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ + + // Build and push app service + def appImageTag = "${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION}" + sh """ + # Build image + docker build -t ${appImageTag} -f app/Dockerfile . || exit 1 + # Push image + docker push ${appImageTag} || exit 1 + """ + } catch (Exception e) { + echo "Error during build/push: ${e.getMessage()}" + throw e + } + } } } } - } -} -} + } // Closing brace for 'stages' block post { success { From ddf1e19f94e604e18325436e16bd6cbbbe612a9e Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 17:22:37 +0530 Subject: [PATCH 31/49] Update Dockerfile --- app/Dockerfile | 49 +++---------------------------------------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/app/Dockerfile b/app/Dockerfile index 02d5b749..f8066db5 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -22,7 +22,6 @@ RUN pnpm install && pnpm run generate FROM base AS release WORKDIR /app - # Install required packages RUN apt-get update && apt-get install -y \ nginx \ @@ -39,51 +38,9 @@ COPY app/nginx /etc/nginx # Create directory for supervisor configs RUN mkdir -p /etc/supervisor/conf.d -# Create entrypoint script -COPY < Date: Mon, 24 Feb 2025 18:01:51 +0530 Subject: [PATCH 32/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 44 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 633b36ed..6b59752c 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -5,18 +5,15 @@ pipeline { args '-u root' } } - parameters { choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') } - environment { ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' AWS_REGION = 'us-east-1' ENV_NAME = "${params.ENVIRONMENT == 'dev' ? 'dampen' : params.ENVIRONMENT == 'qa' ? 'iamdqa' : 'prod'}" } - stages { stage('Install Dependencies') { steps { @@ -31,7 +28,6 @@ pipeline { } } } - stage('Checkout') { steps { cleanWs() @@ -39,10 +35,16 @@ pipeline { url: 'https://github.com/shivani-fission/FloTorch.git' } } - stage('Build and Push Images') { steps { script { + def services = [ + 'app', + 'indexing', + 'retriever', + 'evaluation' + ] + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { try { // ECR Login @@ -51,14 +53,25 @@ pipeline { docker login --username AWS --password-stdin ${ECR_REGISTRY} """ - // Build and push app service - def appImageTag = "${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION}" - sh """ - # Build image - docker build -t ${appImageTag} -f app/Dockerfile . || exit 1 - # Push image - docker push ${appImageTag} || exit 1 - """ + // Build and push each service + services.each { service -> + def imageTag = "${ECR_REGISTRY}/flotorch-${service}-${ENV_NAME}:${params.BUILD_VERSION}" + echo "Building and pushing ${service} image..." + + // Determine which Dockerfile to use + def dockerfilePath = service == 'app' ? 'app/Dockerfile' : 'Dockerfile' + + sh """ + # Build image + docker build -t ${imageTag} \ + --build-arg SERVICE=${service} \ + -f ${dockerfilePath} . || exit 1 + # Push image + docker push ${imageTag} || exit 1 + """ + + echo "${service} image built and pushed successfully" + } } catch (Exception e) { echo "Error during build/push: ${e.getMessage()}" throw e @@ -67,11 +80,10 @@ pipeline { } } } - } // Closing brace for 'stages' block - + } post { success { - echo "Successfully built and pushed images for ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" + echo "Successfully built and pushed all images for ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" } failure { echo "Failed to build/push images for ${params.ENVIRONMENT}" From 32de9730c171a96d2d85f2ce2b43fa037b425621 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 18:18:23 +0530 Subject: [PATCH 33/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 60 ++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 6b59752c..7c4a87a4 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -44,7 +44,7 @@ pipeline { 'retriever', 'evaluation' ] - + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { try { // ECR Login @@ -52,25 +52,56 @@ pipeline { aws ecr get-login-password --region ${AWS_REGION} | \ docker login --username AWS --password-stdin ${ECR_REGISTRY} """ - + // Build and push each service services.each { service -> def imageTag = "${ECR_REGISTRY}/flotorch-${service}-${ENV_NAME}:${params.BUILD_VERSION}" echo "Building and pushing ${service} image..." - - // Determine which Dockerfile to use - def dockerfilePath = service == 'app' ? 'app/Dockerfile' : 'Dockerfile' - + + // Get the correct Dockerfile path and verify it exists + def dockerfilePath + if (service == 'indexing') { + // Check for fargate_indexing.Dockerfile first + if (fileExists("${service}/fargate_indexing.Dockerfile")) { + dockerfilePath = "${service}/fargate_indexing.Dockerfile" + } else if (fileExists("${service}/Dockerfile")) { + dockerfilePath = "${service}/Dockerfile" + } else { + error "No Dockerfile found for ${service}" + } + } else { + dockerfilePath = "${service}/Dockerfile" + if (!fileExists(dockerfilePath)) { + error "Dockerfile not found at ${dockerfilePath}" + } + } + + echo "Using Dockerfile at: ${dockerfilePath}" + + // Create ECR repository if it doesn't exist + def repoName = "flotorch-${service}-${ENV_NAME}" sh """ - # Build image - docker build -t ${imageTag} \ - --build-arg SERVICE=${service} \ - -f ${dockerfilePath} . || exit 1 - # Push image - docker push ${imageTag} || exit 1 + if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then + echo "Creating ECR repository: ${repoName}" + aws ecr create-repository --repository-name ${repoName} --region ${AWS_REGION} + fi """ - - echo "${service} image built and pushed successfully" + + // Build and push with full error handling + try { + sh """ + # Build image + DOCKER_BUILDKIT=1 docker build -t ${imageTag} \ + -f ${dockerfilePath} . || exit 1 + + # Push image + docker push ${imageTag} || exit 1 + """ + echo "${service} image built and pushed successfully" + } catch (Exception e) { + echo "Failed to build/push ${service} image: ${e.getMessage()}" + throw e + } } } catch (Exception e) { echo "Error during build/push: ${e.getMessage()}" @@ -94,3 +125,4 @@ pipeline { } } } + From 0a6a3dc63ed0641427f333447bba915369e3ce4b Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Mon, 24 Feb 2025 18:22:30 +0530 Subject: [PATCH 34/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 7c4a87a4..5afceb06 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -1,3 +1,4 @@ + pipeline { agent { docker { @@ -59,21 +60,9 @@ pipeline { echo "Building and pushing ${service} image..." // Get the correct Dockerfile path and verify it exists - def dockerfilePath - if (service == 'indexing') { - // Check for fargate_indexing.Dockerfile first - if (fileExists("${service}/fargate_indexing.Dockerfile")) { - dockerfilePath = "${service}/fargate_indexing.Dockerfile" - } else if (fileExists("${service}/Dockerfile")) { - dockerfilePath = "${service}/Dockerfile" - } else { - error "No Dockerfile found for ${service}" - } - } else { - dockerfilePath = "${service}/Dockerfile" - if (!fileExists(dockerfilePath)) { - error "Dockerfile not found at ${dockerfilePath}" - } + def dockerfilePath = "${service}/Dockerfile" + if (!fileExists(dockerfilePath)) { + error "Dockerfile not found at ${dockerfilePath}" } echo "Using Dockerfile at: ${dockerfilePath}" @@ -87,12 +76,11 @@ pipeline { fi """ - // Build and push with full error handling + // Build and push try { sh """ # Build image - DOCKER_BUILDKIT=1 docker build -t ${imageTag} \ - -f ${dockerfilePath} . || exit 1 + docker build -t ${imageTag} -f ${dockerfilePath} . || exit 1 # Push image docker push ${imageTag} || exit 1 @@ -125,4 +113,3 @@ pipeline { } } } - From 782426f881957337cde559ddb802ef72b33baa84 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 12:32:30 +0530 Subject: [PATCH 35/49] Create Flotorch-Deployments --- Jenkins/Flotorch-Deployments | 153 +++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 Jenkins/Flotorch-Deployments diff --git a/Jenkins/Flotorch-Deployments b/Jenkins/Flotorch-Deployments new file mode 100644 index 00000000..098f39dc --- /dev/null +++ b/Jenkins/Flotorch-Deployments @@ -0,0 +1,153 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + parameters { + choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') + string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') + } + environment { + ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' + AWS_REGION = 'us-east-1' + ENV_NAME = "${params.ENVIRONMENT == 'dev' ? 'dampen' : params.ENVIRONMENT == 'qa' ? 'iamdqa' : 'prod'}" + } + stages { + stage('Install Dependencies') { + steps { + script { + sh ''' + apt-get update + apt-get install -y docker.io curl unzip git + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + ''' + } + } + } + stage('Checkout') { + steps { + cleanWs() + git branch: 'main', + url: 'https://github.com/shivani-fission/FloTorch.git' + } + } + stage("Build and Push Images - ${params.ENVIRONMENT.toUpperCase()}") { + steps { + script { + def services = [ + 'app', + 'indexing', + 'retriever', + 'evaluation' + ] + + def builtImages = [] + + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + try { + // ECR Login + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ + + // Build and push each service + services.each { service -> + def repoName = "flotorch-${service}-${ENV_NAME}" + def imageTag = "${ECR_REGISTRY}/${repoName}:${params.BUILD_VERSION}" + echo "Building and pushing ${service} image for ${params.ENVIRONMENT}..." + + // Get the correct Dockerfile path and verify it exists + def dockerfilePath = "${service}/Dockerfile" + if (!fileExists(dockerfilePath)) { + error "Dockerfile not found at ${dockerfilePath}" + } + + echo "Using Dockerfile at: ${dockerfilePath}" + + // Create ECR repository if it doesn't exist + sh """ + if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then + echo "Creating ECR repository: ${repoName}" + aws ecr create-repository --repository-name ${repoName} --region ${AWS_REGION} + fi + """ + + // Build and push + try { + sh """ + # Build image + docker build -t ${imageTag} -f ${dockerfilePath} . || exit 1 + + # Push image + docker push ${imageTag} || exit 1 + """ + builtImages.add([ + service: service, + repo: repoName, + image: imageTag, + environment: params.ENVIRONMENT + ]) + echo "${service} image built and pushed to ${params.ENVIRONMENT} successfully" + } catch (Exception e) { + echo "Failed to build/push ${service} image for ${params.ENVIRONMENT}: ${e.getMessage()}" + throw e + } + } + + // Create dashboard summary + def summary = "Images pushed to ${params.ENVIRONMENT.toUpperCase()} repositories:\n" + builtImages.each { img -> + summary += "- ${img.service}: ${img.image}\n" + } + + // Display dashboard summary + echo summary + + // Create a build artifact with the summary + writeFile file: "${params.ENVIRONMENT}-images.txt", text: summary + archiveArtifacts artifacts: "${params.ENVIRONMENT}-images.txt", allowEmptyArchive: true + + } catch (Exception e) { + echo "Error during build/push for ${params.ENVIRONMENT}: ${e.getMessage()}" + throw e + } + } + } + } + } + stage('Deployment Summary') { + steps { + script { + echo """ + ================================================= + DEPLOYMENT SUMMARY FOR ${params.ENVIRONMENT.toUpperCase()} + ================================================= + Environment: ${params.ENVIRONMENT} + Env Name: ${ENV_NAME} + Build Version: ${params.BUILD_VERSION} + Registry: ${ECR_REGISTRY} + Services Deployed: app, indexing, retriever, evaluation + ================================================= + """ + } + } + } + } + post { + success { + echo "Successfully built and pushed all images for ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" + } + failure { + echo "Failed to build/push images for ${params.ENVIRONMENT}" + } + always { + cleanWs() + sh 'docker system prune -f' + } + } +} From 72d77af1c774349c19198489e3c835dfd5c740a8 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 12:38:24 +0530 Subject: [PATCH 36/49] Delete Jenkins/Flotorch-Deployments --- Jenkins/Flotorch-Deployments | 153 ----------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 Jenkins/Flotorch-Deployments diff --git a/Jenkins/Flotorch-Deployments b/Jenkins/Flotorch-Deployments deleted file mode 100644 index 098f39dc..00000000 --- a/Jenkins/Flotorch-Deployments +++ /dev/null @@ -1,153 +0,0 @@ -pipeline { - agent { - docker { - image 'ubuntu:latest' - args '-u root' - } - } - parameters { - choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') - string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') - } - environment { - ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' - AWS_REGION = 'us-east-1' - ENV_NAME = "${params.ENVIRONMENT == 'dev' ? 'dampen' : params.ENVIRONMENT == 'qa' ? 'iamdqa' : 'prod'}" - } - stages { - stage('Install Dependencies') { - steps { - script { - sh ''' - apt-get update - apt-get install -y docker.io curl unzip git - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -o -q awscliv2.zip - ./aws/install --update - ''' - } - } - } - stage('Checkout') { - steps { - cleanWs() - git branch: 'main', - url: 'https://github.com/shivani-fission/FloTorch.git' - } - } - stage("Build and Push Images - ${params.ENVIRONMENT.toUpperCase()}") { - steps { - script { - def services = [ - 'app', - 'indexing', - 'retriever', - 'evaluation' - ] - - def builtImages = [] - - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - try { - // ECR Login - sh """ - aws ecr get-login-password --region ${AWS_REGION} | \ - docker login --username AWS --password-stdin ${ECR_REGISTRY} - """ - - // Build and push each service - services.each { service -> - def repoName = "flotorch-${service}-${ENV_NAME}" - def imageTag = "${ECR_REGISTRY}/${repoName}:${params.BUILD_VERSION}" - echo "Building and pushing ${service} image for ${params.ENVIRONMENT}..." - - // Get the correct Dockerfile path and verify it exists - def dockerfilePath = "${service}/Dockerfile" - if (!fileExists(dockerfilePath)) { - error "Dockerfile not found at ${dockerfilePath}" - } - - echo "Using Dockerfile at: ${dockerfilePath}" - - // Create ECR repository if it doesn't exist - sh """ - if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then - echo "Creating ECR repository: ${repoName}" - aws ecr create-repository --repository-name ${repoName} --region ${AWS_REGION} - fi - """ - - // Build and push - try { - sh """ - # Build image - docker build -t ${imageTag} -f ${dockerfilePath} . || exit 1 - - # Push image - docker push ${imageTag} || exit 1 - """ - builtImages.add([ - service: service, - repo: repoName, - image: imageTag, - environment: params.ENVIRONMENT - ]) - echo "${service} image built and pushed to ${params.ENVIRONMENT} successfully" - } catch (Exception e) { - echo "Failed to build/push ${service} image for ${params.ENVIRONMENT}: ${e.getMessage()}" - throw e - } - } - - // Create dashboard summary - def summary = "Images pushed to ${params.ENVIRONMENT.toUpperCase()} repositories:\n" - builtImages.each { img -> - summary += "- ${img.service}: ${img.image}\n" - } - - // Display dashboard summary - echo summary - - // Create a build artifact with the summary - writeFile file: "${params.ENVIRONMENT}-images.txt", text: summary - archiveArtifacts artifacts: "${params.ENVIRONMENT}-images.txt", allowEmptyArchive: true - - } catch (Exception e) { - echo "Error during build/push for ${params.ENVIRONMENT}: ${e.getMessage()}" - throw e - } - } - } - } - } - stage('Deployment Summary') { - steps { - script { - echo """ - ================================================= - DEPLOYMENT SUMMARY FOR ${params.ENVIRONMENT.toUpperCase()} - ================================================= - Environment: ${params.ENVIRONMENT} - Env Name: ${ENV_NAME} - Build Version: ${params.BUILD_VERSION} - Registry: ${ECR_REGISTRY} - Services Deployed: app, indexing, retriever, evaluation - ================================================= - """ - } - } - } - } - post { - success { - echo "Successfully built and pushed all images for ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" - } - failure { - echo "Failed to build/push images for ${params.ENVIRONMENT}" - } - always { - cleanWs() - sh 'docker system prune -f' - } - } -} From 6008389592caac05f09f7415afe5f47ce9547e2d Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 12:38:45 +0530 Subject: [PATCH 37/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 54 +++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 5afceb06..098f39dc 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -1,4 +1,3 @@ - pipeline { agent { docker { @@ -36,7 +35,7 @@ pipeline { url: 'https://github.com/shivani-fission/FloTorch.git' } } - stage('Build and Push Images') { + stage("Build and Push Images - ${params.ENVIRONMENT.toUpperCase()}") { steps { script { def services = [ @@ -46,6 +45,8 @@ pipeline { 'evaluation' ] + def builtImages = [] + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { try { // ECR Login @@ -56,8 +57,9 @@ pipeline { // Build and push each service services.each { service -> - def imageTag = "${ECR_REGISTRY}/flotorch-${service}-${ENV_NAME}:${params.BUILD_VERSION}" - echo "Building and pushing ${service} image..." + def repoName = "flotorch-${service}-${ENV_NAME}" + def imageTag = "${ECR_REGISTRY}/${repoName}:${params.BUILD_VERSION}" + echo "Building and pushing ${service} image for ${params.ENVIRONMENT}..." // Get the correct Dockerfile path and verify it exists def dockerfilePath = "${service}/Dockerfile" @@ -68,7 +70,6 @@ pipeline { echo "Using Dockerfile at: ${dockerfilePath}" // Create ECR repository if it doesn't exist - def repoName = "flotorch-${service}-${ENV_NAME}" sh """ if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then echo "Creating ECR repository: ${repoName}" @@ -85,20 +86,57 @@ pipeline { # Push image docker push ${imageTag} || exit 1 """ - echo "${service} image built and pushed successfully" + builtImages.add([ + service: service, + repo: repoName, + image: imageTag, + environment: params.ENVIRONMENT + ]) + echo "${service} image built and pushed to ${params.ENVIRONMENT} successfully" } catch (Exception e) { - echo "Failed to build/push ${service} image: ${e.getMessage()}" + echo "Failed to build/push ${service} image for ${params.ENVIRONMENT}: ${e.getMessage()}" throw e } } + + // Create dashboard summary + def summary = "Images pushed to ${params.ENVIRONMENT.toUpperCase()} repositories:\n" + builtImages.each { img -> + summary += "- ${img.service}: ${img.image}\n" + } + + // Display dashboard summary + echo summary + + // Create a build artifact with the summary + writeFile file: "${params.ENVIRONMENT}-images.txt", text: summary + archiveArtifacts artifacts: "${params.ENVIRONMENT}-images.txt", allowEmptyArchive: true + } catch (Exception e) { - echo "Error during build/push: ${e.getMessage()}" + echo "Error during build/push for ${params.ENVIRONMENT}: ${e.getMessage()}" throw e } } } } } + stage('Deployment Summary') { + steps { + script { + echo """ + ================================================= + DEPLOYMENT SUMMARY FOR ${params.ENVIRONMENT.toUpperCase()} + ================================================= + Environment: ${params.ENVIRONMENT} + Env Name: ${ENV_NAME} + Build Version: ${params.BUILD_VERSION} + Registry: ${ECR_REGISTRY} + Services Deployed: app, indexing, retriever, evaluation + ================================================= + """ + } + } + } } post { success { From 6ae3b2ae5695565f9eea7a727ae4b3c7da551bdc Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 12:41:10 +0530 Subject: [PATCH 38/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 098f39dc..e9b90d85 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -35,16 +35,10 @@ pipeline { url: 'https://github.com/shivani-fission/FloTorch.git' } } - stage("Build and Push Images - ${params.ENVIRONMENT.toUpperCase()}") { + stage('Build and Push Images') { steps { script { - def services = [ - 'app', - 'indexing', - 'retriever', - 'evaluation' - ] - + def services = ['app', 'indexing', 'retriever', 'evaluation'] def builtImages = [] withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { @@ -81,10 +75,9 @@ pipeline { try { sh """ # Build image - docker build -t ${imageTag} -f ${dockerfilePath} . || exit 1 - + docker build -t ${imageTag} -f ${dockerfilePath} . # Push image - docker push ${imageTag} || exit 1 + docker push ${imageTag} """ builtImages.add([ service: service, From 471064341a629edac5d16ea236853b903f84280c Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 12:58:09 +0530 Subject: [PATCH 39/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index e9b90d85..0555ff25 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -35,12 +35,25 @@ pipeline { url: 'https://github.com/shivani-fission/FloTorch.git' } } - stage('Build and Push Images') { + stage('Build and Push Images') { steps { script { - def services = ['app', 'indexing', 'retriever', 'evaluation'] + def services = [ + 'app', + 'indexing', + 'retriever', + 'evaluation' + ] + def builtImages = [] + // Display environment-specific header in console + echo """ + =================================================== + BUILDING IMAGES FOR: ${params.ENVIRONMENT.toUpperCase()} + =================================================== + """ + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { try { // ECR Login @@ -75,9 +88,10 @@ pipeline { try { sh """ # Build image - docker build -t ${imageTag} -f ${dockerfilePath} . + docker build -t ${imageTag} -f ${dockerfilePath} . || exit 1 + # Push image - docker push ${imageTag} + docker push ${imageTag} || exit 1 """ builtImages.add([ service: service, @@ -105,6 +119,9 @@ pipeline { writeFile file: "${params.ENVIRONMENT}-images.txt", text: summary archiveArtifacts artifacts: "${params.ENVIRONMENT}-images.txt", allowEmptyArchive: true + // Add environment information to build description + currentBuild.description = "ENV: ${params.ENVIRONMENT.toUpperCase()} | Version: ${params.BUILD_VERSION}" + } catch (Exception e) { echo "Error during build/push for ${params.ENVIRONMENT}: ${e.getMessage()}" throw e @@ -113,7 +130,7 @@ pipeline { } } } - stage('Deployment Summary') { + stage('Display Environment Dashboard') { steps { script { echo """ @@ -125,8 +142,12 @@ pipeline { Build Version: ${params.BUILD_VERSION} Registry: ${ECR_REGISTRY} Services Deployed: app, indexing, retriever, evaluation + Repository Pattern: flotorch-{service}-${ENV_NAME} ================================================= """ + + // Set the build display name to show environment + currentBuild.displayName = "#${BUILD_NUMBER} - ${params.ENVIRONMENT.toUpperCase()}" } } } From cdc9553da546e5e3f1e65d14a7d63c947e7b0339 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 13:12:51 +0530 Subject: [PATCH 40/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 40 +++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 0555ff25..a48a2e3e 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -38,19 +38,17 @@ pipeline { stage('Build and Push Images') { steps { script { - def services = [ - 'app', - 'indexing', - 'retriever', - 'evaluation' - ] + // Always build all four services + def services = ['app', 'indexing', 'retriever', 'evaluation'] def builtImages = [] + def failedImages = [] // Display environment-specific header in console echo """ =================================================== BUILDING IMAGES FOR: ${params.ENVIRONMENT.toUpperCase()} + ALL SERVICES: ${services.join(', ')} =================================================== """ @@ -70,8 +68,19 @@ pipeline { // Get the correct Dockerfile path and verify it exists def dockerfilePath = "${service}/Dockerfile" + + // Check if Dockerfile exists if (!fileExists(dockerfilePath)) { - error "Dockerfile not found at ${dockerfilePath}" + echo "WARNING: Dockerfile not found at ${dockerfilePath}, creating empty file for testing" + // Create an empty Dockerfile for testing purposes + sh "mkdir -p ${service}" + sh """ + cat > ${dockerfilePath} << 'EOL' +FROM ubuntu:latest +RUN echo "This is a test image for ${service}" +CMD ["echo", "Hello from ${service}"] +EOL + """ } echo "Using Dockerfile at: ${dockerfilePath}" @@ -102,10 +111,15 @@ pipeline { echo "${service} image built and pushed to ${params.ENVIRONMENT} successfully" } catch (Exception e) { echo "Failed to build/push ${service} image for ${params.ENVIRONMENT}: ${e.getMessage()}" - throw e + failedImages.add(service) } } + // Check if any images failed + if (failedImages.size() > 0) { + error "Failed to build/push the following images: ${failedImages.join(', ')}" + } + // Create dashboard summary def summary = "Images pushed to ${params.ENVIRONMENT.toUpperCase()} repositories:\n" builtImages.each { img -> @@ -120,7 +134,7 @@ pipeline { archiveArtifacts artifacts: "${params.ENVIRONMENT}-images.txt", allowEmptyArchive: true // Add environment information to build description - currentBuild.description = "ENV: ${params.ENVIRONMENT.toUpperCase()} | Version: ${params.BUILD_VERSION}" + currentBuild.description = "ENV: ${params.ENVIRONMENT.toUpperCase()} | Version: ${params.BUILD_VERSION} | Services: ${builtImages.size()}/4" } catch (Exception e) { echo "Error during build/push for ${params.ENVIRONMENT}: ${e.getMessage()}" @@ -141,7 +155,13 @@ pipeline { Env Name: ${ENV_NAME} Build Version: ${params.BUILD_VERSION} Registry: ${ECR_REGISTRY} - Services Deployed: app, indexing, retriever, evaluation + + Services Deployed: + - app: flotorch-app-${ENV_NAME} + - indexing: flotorch-indexing-${ENV_NAME} + - retriever: flotorch-retriever-${ENV_NAME} + - evaluation: flotorch-evaluation-${ENV_NAME} + Repository Pattern: flotorch-{service}-${ENV_NAME} ================================================= """ From eb6e2df489643a1d02d8551fb46e8b4c89ffab3a Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 13:26:33 +0530 Subject: [PATCH 41/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 37 +++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index a48a2e3e..cabe57d2 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -48,6 +48,7 @@ pipeline { echo """ =================================================== BUILDING IMAGES FOR: ${params.ENVIRONMENT.toUpperCase()} + BUILD VERSION: ${params.BUILD_VERSION} ALL SERVICES: ${services.join(', ')} =================================================== """ @@ -63,7 +64,9 @@ pipeline { // Build and push each service services.each { service -> def repoName = "flotorch-${service}-${ENV_NAME}" - def imageTag = "${ECR_REGISTRY}/${repoName}:${params.BUILD_VERSION}" + def imageTagWithVersion = "${ECR_REGISTRY}/${repoName}:${params.BUILD_VERSION}" + def imageTagLatest = "${ECR_REGISTRY}/${repoName}:latest" + echo "Building and pushing ${service} image for ${params.ENVIRONMENT}..." // Get the correct Dockerfile path and verify it exists @@ -97,15 +100,31 @@ EOL try { sh """ # Build image - docker build -t ${imageTag} -f ${dockerfilePath} . || exit 1 + docker build -t ${imageTagWithVersion} -f ${dockerfilePath} . || exit 1 + + # Tag with version number only (no additional tags) + docker push ${imageTagWithVersion} || exit 1 + + # Only tag as 'latest' if specifically requested + if [ "${params.BUILD_VERSION}" != "latest" ]; then + docker tag ${imageTagWithVersion} ${imageTagLatest} + docker push ${imageTagLatest} || exit 1 + echo "Tagged and pushed as both ${params.BUILD_VERSION} and latest" + else + echo "Tagged and pushed as latest only" + fi - # Push image - docker push ${imageTag} || exit 1 + # Remove any old multi-tag references + if aws ecr list-images --repository-name ${repoName} --query "imageIds[?contains(imageTag, ',')].imageTag" --output text | grep -q '.'; then + echo "Found multi-tagged images, removing them..." + aws ecr list-images --repository-name ${repoName} --query "imageIds[?contains(imageTag, ',')].imageTag" --output text | xargs -I {} aws ecr batch-delete-image --repository-name ${repoName} --image-ids imageTag={} + fi """ + builtImages.add([ service: service, repo: repoName, - image: imageTag, + image: imageTagWithVersion, environment: params.ENVIRONMENT ]) echo "${service} image built and pushed to ${params.ENVIRONMENT} successfully" @@ -157,10 +176,10 @@ EOL Registry: ${ECR_REGISTRY} Services Deployed: - - app: flotorch-app-${ENV_NAME} - - indexing: flotorch-indexing-${ENV_NAME} - - retriever: flotorch-retriever-${ENV_NAME} - - evaluation: flotorch-evaluation-${ENV_NAME} + - app: ${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION} + - indexing: ${ECR_REGISTRY}/flotorch-indexing-${ENV_NAME}:${params.BUILD_VERSION} + - retriever: ${ECR_REGISTRY}/flotorch-retriever-${ENV_NAME}:${params.BUILD_VERSION} + - evaluation: ${ECR_REGISTRY}/flotorch-evaluation-${ENV_NAME}:${params.BUILD_VERSION} Repository Pattern: flotorch-{service}-${ENV_NAME} ================================================= From ed49d031a8670d381df859b865aa0f9ccf3abfdf Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 13:57:23 +0530 Subject: [PATCH 42/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 70 ++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index cabe57d2..8fb266e8 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -8,6 +8,7 @@ pipeline { parameters { choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') + booleanParam(name: 'FORCE_REBUILD', defaultValue: true, description: 'Force rebuild even if image exists') } environment { ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' @@ -20,7 +21,7 @@ pipeline { script { sh ''' apt-get update - apt-get install -y docker.io curl unzip git + apt-get install -y docker.io curl unzip git jq curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip -o -q awscliv2.zip ./aws/install --update @@ -35,6 +36,54 @@ pipeline { url: 'https://github.com/shivani-fission/FloTorch.git' } } + stage('Clean Up Old Tags') { + steps { + script { + // Define services + def services = ['app', 'indexing', 'retriever', 'evaluation'] + + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + services.each { service -> + def repoName = "flotorch-${service}-${ENV_NAME}" + + echo "Cleaning up repository: ${repoName}" + + try { + // Check if repository exists + sh """ + if aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then + echo "Repository exists: ${repoName}" + + # First, remove any comma-separated multi-tagged images + MULTI_TAGS=\$(aws ecr list-images --repository-name ${repoName} --query "imageIds[?contains(imageTag, ',')].imageTag" --output json) + if [ "\$(echo \$MULTI_TAGS | jq 'length')" -gt "0" ]; then + echo "Found multi-tagged images, removing them..." + echo \$MULTI_TAGS | jq -r '.[]' | while read -r tag; do + aws ecr batch-delete-image --repository-name ${repoName} --image-ids imageTag="\$tag" || true + done + fi + + # If FORCE_REBUILD is true, remove the specific version tag if it exists + if [ "${params.FORCE_REBUILD}" = "true" ]; then + echo "Checking if version ${params.BUILD_VERSION} exists..." + if aws ecr describe-images --repository-name ${repoName} --image-ids imageTag=${params.BUILD_VERSION} 2>/dev/null; then + echo "Removing existing image with tag ${params.BUILD_VERSION}" + aws ecr batch-delete-image --repository-name ${repoName} --image-ids imageTag=${params.BUILD_VERSION} || true + fi + fi + else + echo "Repository does not exist: ${repoName}" + fi + """ + } catch (Exception e) { + echo "Warning: Error during cleanup for ${repoName}: ${e.getMessage()}" + // Continue with next repository + } + } + } + } + } + } stage('Build and Push Images') { steps { script { @@ -80,7 +129,7 @@ pipeline { sh """ cat > ${dockerfilePath} << 'EOL' FROM ubuntu:latest -RUN echo "This is a test image for ${service}" +RUN echo "This is a test image for ${service} - built on $(date)" CMD ["echo", "Hello from ${service}"] EOL """ @@ -98,14 +147,17 @@ EOL // Build and push try { + // Add a timestamp to the image to ensure it's unique + def timestamp = new Date().format("yyyyMMdd-HHmmss") + sh """ - # Build image - docker build -t ${imageTagWithVersion} -f ${dockerfilePath} . || exit 1 + # Build image with build date label to ensure uniqueness + docker build --build-arg BUILD_DATE=${timestamp} -t ${imageTagWithVersion} -f ${dockerfilePath} . || exit 1 - # Tag with version number only (no additional tags) + # Push with version number docker push ${imageTagWithVersion} || exit 1 - # Only tag as 'latest' if specifically requested + # Tag as latest if requested if [ "${params.BUILD_VERSION}" != "latest" ]; then docker tag ${imageTagWithVersion} ${imageTagLatest} docker push ${imageTagLatest} || exit 1 @@ -113,12 +165,6 @@ EOL else echo "Tagged and pushed as latest only" fi - - # Remove any old multi-tag references - if aws ecr list-images --repository-name ${repoName} --query "imageIds[?contains(imageTag, ',')].imageTag" --output text | grep -q '.'; then - echo "Found multi-tagged images, removing them..." - aws ecr list-images --repository-name ${repoName} --query "imageIds[?contains(imageTag, ',')].imageTag" --output text | xargs -I {} aws ecr batch-delete-image --repository-name ${repoName} --image-ids imageTag={} - fi """ builtImages.add([ From 810754c06980f5b925e8f0bb91fc728da46a330a Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 14:01:40 +0530 Subject: [PATCH 43/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 8fb266e8..44ca7805 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -129,7 +129,7 @@ pipeline { sh """ cat > ${dockerfilePath} << 'EOL' FROM ubuntu:latest -RUN echo "This is a test image for ${service} - built on $(date)" +RUN echo "This is a test image for ${service} - built on \$(date)" CMD ["echo", "Hello from ${service}"] EOL """ @@ -152,7 +152,7 @@ EOL sh """ # Build image with build date label to ensure uniqueness - docker build --build-arg BUILD_DATE=${timestamp} -t ${imageTagWithVersion} -f ${dockerfilePath} . || exit 1 + docker build --build-arg BUILD_DATE="${timestamp}" -t ${imageTagWithVersion} -f ${dockerfilePath} . || exit 1 # Push with version number docker push ${imageTagWithVersion} || exit 1 From 1fe0b94fb8f9732420ef72c3d87cc011292be203 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 14:13:20 +0530 Subject: [PATCH 44/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 44ca7805..6dfdffbc 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -8,7 +8,7 @@ pipeline { parameters { choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') - booleanParam(name: 'FORCE_REBUILD', defaultValue: true, description: 'Force rebuild even if image exists') + booleanParam(name: 'FORCE_REBUILD', defaultValue: true, description: 'Force rebuild even if image with the same tag exists') } environment { ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' @@ -54,7 +54,7 @@ pipeline { if aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then echo "Repository exists: ${repoName}" - # First, remove any comma-separated multi-tagged images + # Only remove comma-separated multi-tagged images MULTI_TAGS=\$(aws ecr list-images --repository-name ${repoName} --query "imageIds[?contains(imageTag, ',')].imageTag" --output json) if [ "\$(echo \$MULTI_TAGS | jq 'length')" -gt "0" ]; then echo "Found multi-tagged images, removing them..." @@ -63,7 +63,7 @@ pipeline { done fi - # If FORCE_REBUILD is true, remove the specific version tag if it exists + # Only if FORCE_REBUILD is true, remove the specific version tag if it exists if [ "${params.FORCE_REBUILD}" = "true" ]; then echo "Checking if version ${params.BUILD_VERSION} exists..." if aws ecr describe-images --repository-name ${repoName} --image-ids imageTag=${params.BUILD_VERSION} 2>/dev/null; then @@ -154,17 +154,10 @@ EOL # Build image with build date label to ensure uniqueness docker build --build-arg BUILD_DATE="${timestamp}" -t ${imageTagWithVersion} -f ${dockerfilePath} . || exit 1 - # Push with version number + # Push with version number only - no latest tag docker push ${imageTagWithVersion} || exit 1 - # Tag as latest if requested - if [ "${params.BUILD_VERSION}" != "latest" ]; then - docker tag ${imageTagWithVersion} ${imageTagLatest} - docker push ${imageTagLatest} || exit 1 - echo "Tagged and pushed as both ${params.BUILD_VERSION} and latest" - else - echo "Tagged and pushed as latest only" - fi + echo "Tagged and pushed as ${params.BUILD_VERSION} only" """ builtImages.add([ From eb37a94f1551961efcc7633d0cfe054805beccce Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Tue, 25 Feb 2025 14:25:30 +0530 Subject: [PATCH 45/49] Update Jenkinsfile.deployment --- Jenkins/Jenkinsfile.deployment | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment index 6dfdffbc..8860dbfe 100644 --- a/Jenkins/Jenkinsfile.deployment +++ b/Jenkins/Jenkinsfile.deployment @@ -149,12 +149,17 @@ EOL try { // Add a timestamp to the image to ensure it's unique def timestamp = new Date().format("yyyyMMdd-HHmmss") + def uniqueImageTag = "${ECR_REGISTRY}/${repoName}:build-${timestamp}" sh """ - # Build image with build date label to ensure uniqueness - docker build --build-arg BUILD_DATE="${timestamp}" -t ${imageTagWithVersion} -f ${dockerfilePath} . || exit 1 + # Build image with a unique temporary tag first to avoid reusing existing images + echo "Building fresh image with timestamp ${timestamp}" + docker build --no-cache --build-arg BUILD_DATE="${timestamp}" -t ${uniqueImageTag} -f ${dockerfilePath} . || exit 1 - # Push with version number only - no latest tag + # Tag with the requested version + docker tag ${uniqueImageTag} ${imageTagWithVersion} + + # Push the version-tagged image docker push ${imageTagWithVersion} || exit 1 echo "Tagged and pushed as ${params.BUILD_VERSION} only" From 59049fd830e703dc8a2d42912013e8c89f222f55 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 26 Feb 2025 11:06:53 +0530 Subject: [PATCH 46/49] Create Jenkinsfile.imagepush --- Jenkins/Jenkinsfile.imagepush | 1 + 1 file changed, 1 insertion(+) create mode 100644 Jenkins/Jenkinsfile.imagepush diff --git a/Jenkins/Jenkinsfile.imagepush b/Jenkins/Jenkinsfile.imagepush new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/Jenkins/Jenkinsfile.imagepush @@ -0,0 +1 @@ + From 2bfceedf9cc1d4329379b75cd92bc1d35bd0c538 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 26 Feb 2025 11:11:20 +0530 Subject: [PATCH 47/49] Update Jenkinsfile.imagepush --- Jenkins/Jenkinsfile.imagepush | 161 +++++++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/Jenkins/Jenkinsfile.imagepush b/Jenkins/Jenkinsfile.imagepush index 8b137891..a2cb211d 100644 --- a/Jenkins/Jenkinsfile.imagepush +++ b/Jenkins/Jenkinsfile.imagepush @@ -1 +1,160 @@ - +pipeline { + agent { + docker { + image 'public.ecr.aws/amazonlinux/amazonlinux:2' + args '-u root' + } + } + + parameters { + string(name: 'ENVIRONMENT_SUFFIX', defaultValue: '', description: 'Enter the environment suffix for ECR repositories (e.g., jasper)') + } + + environment { + ECR_ACCOUNT_ID = '677276078734' + AWS_REGION = 'us-east-1' + ENV_SUFFIX = "${params.ENVIRONMENT_SUFFIX}" + } + + stages { + stage('Validate Parameters') { + steps { + script { + if (params.ENVIRONMENT_SUFFIX.trim() == '') { + error "ENVIRONMENT_SUFFIX parameter is required. Please provide a valid value." + } + echo "Building and pushing images with suffix: ${ENV_SUFFIX}" + } + } + } + + stage('Install Dependencies') { + steps { + sh ''' + yum update -y + yum install -y docker git curl unzip + amazon-linux-extras install docker -y + service docker start + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + ''' + } + } + + stage('ECR Login') { + steps { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh ''' + # Get AWS account ID + account_id=$(aws sts get-caller-identity --query Account --output text) + + # Login to ECR + aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com + ''' + } + } + } + + stage('Build and Push Images') { + steps { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh ''' + # Get AWS account ID + account_id=$(aws sts get-caller-identity --query Account --output text) + + # Verify and/or create ECR repositories + repos=("flotorch-app" "flotorch-indexing" "flotorch-retriever" "flotorch-evaluation" "flotorch-runtime" "flotorch-costcompute") + + for repo in "${repos[@]}"; do + repo_name="${repo}-${ENV_SUFFIX}" + if ! aws ecr describe-repositories --repository-names "$repo_name" --region ${AWS_REGION} >/dev/null 2>&1; then + echo "Creating repository: $repo_name" + aws ecr create-repository --repository-name "$repo_name" --region ${AWS_REGION} --image-scanning-configuration scanOnPush=true + else + echo "Repository $repo_name already exists" + fi + done + + # Build and push Docker images + echo "Building and pushing Docker images with suffix: ${ENV_SUFFIX}" + + # Build app image + echo "Building app image..." + docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-app-${ENV_SUFFIX}:latest -f app/Dockerfile . + docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-app-${ENV_SUFFIX}:latest + + # Build indexing image + echo "Building indexing image..." + docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-indexing-${ENV_SUFFIX}:latest -f indexing/fargate_indexing.Dockerfile . + docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-indexing-${ENV_SUFFIX}:latest + + # Build retriever image + echo "Building retriever image..." + docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-retriever-${ENV_SUFFIX}:latest -f retriever/fargate_retriever.Dockerfile . + docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-retriever-${ENV_SUFFIX}:latest + + # Build evaluation image + echo "Building evaluation image..." + docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-evaluation-${ENV_SUFFIX}:latest -f evaluation/fargate_evaluation.Dockerfile . + docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-evaluation-${ENV_SUFFIX}:latest + + # Build runtime image + echo "Building runtime image..." + docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-runtime-${ENV_SUFFIX}:latest -f opensearch/opensearch.Dockerfile . + docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-runtime-${ENV_SUFFIX}:latest + + # Build cost compute image + echo "Building cost compute image..." + cd lambda_handlers + docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-costcompute-${ENV_SUFFIX}:latest -f cost_handler/Dockerfile . + docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-costcompute-${ENV_SUFFIX}:latest + cd .. + ''' + } + } + } + + stage('Display Build Summary') { + steps { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh ''' + # Get AWS account ID + account_id=$(aws sts get-caller-identity --query Account --output text) + + echo "===================================================" + echo "BUILD AND PUSH SUMMARY FOR ${ENV_SUFFIX}" + echo "===================================================" + echo "Environment Suffix: ${ENV_SUFFIX}" + echo "AWS Region: ${AWS_REGION}" + echo "ECR Account ID: ${account_id}" + echo "" + echo "Images Pushed:" + echo "- flotorch-app-${ENV_SUFFIX}:latest" + echo "- flotorch-indexing-${ENV_SUFFIX}:latest" + echo "- flotorch-retriever-${ENV_SUFFIX}:latest" + echo "- flotorch-evaluation-${ENV_SUFFIX}:latest" + echo "- flotorch-runtime-${ENV_SUFFIX}:latest" + echo "- flotorch-costcompute-${ENV_SUFFIX}:latest" + echo "" + echo "Repository Pattern: flotorch-{service}-${ENV_SUFFIX}" + echo "===================================================" + ''' + } + } + } + } + + post { + success { + echo "All images successfully built and pushed to ECR repositories with suffix: ${ENV_SUFFIX}" + } + failure { + echo "Failed to build and push images to ECR repositories" + } + always { + cleanWs() + sh 'docker system prune -f' + } + } +} From 29b1df2732d401c251e10358316f02ee7177eb3e Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Wed, 26 Feb 2025 11:21:05 +0530 Subject: [PATCH 48/49] Update Jenkinsfile.imagepush --- Jenkins/Jenkinsfile.imagepush | 199 ++++++++++++++++++---------------- 1 file changed, 104 insertions(+), 95 deletions(-) diff --git a/Jenkins/Jenkinsfile.imagepush b/Jenkins/Jenkinsfile.imagepush index a2cb211d..2ad5cff6 100644 --- a/Jenkins/Jenkinsfile.imagepush +++ b/Jenkins/Jenkinsfile.imagepush @@ -1,21 +1,18 @@ pipeline { agent { docker { - image 'public.ecr.aws/amazonlinux/amazonlinux:2' + image 'ubuntu:latest' args '-u root' } } - parameters { string(name: 'ENVIRONMENT_SUFFIX', defaultValue: '', description: 'Enter the environment suffix for ECR repositories (e.g., jasper)') } - environment { - ECR_ACCOUNT_ID = '677276078734' + ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' AWS_REGION = 'us-east-1' ENV_SUFFIX = "${params.ENVIRONMENT_SUFFIX}" } - stages { stage('Validate Parameters') { steps { @@ -31,10 +28,8 @@ pipeline { stage('Install Dependencies') { steps { sh ''' - yum update -y - yum install -y docker git curl unzip - amazon-linux-extras install docker -y - service docker start + apt-get update + apt-get install -y docker.io curl unzip git curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip -o -q awscliv2.zip ./aws/install --update @@ -42,104 +37,118 @@ pipeline { } } - stage('ECR Login') { + stage('Checkout') { steps { - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - sh ''' - # Get AWS account ID - account_id=$(aws sts get-caller-identity --query Account --output text) - - # Login to ECR - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com - ''' - } + cleanWs() + git branch: 'main', + url: 'https://github.com/shivani-fission/FloTorch.git' } } stage('Build and Push Images') { steps { - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - sh ''' - # Get AWS account ID - account_id=$(aws sts get-caller-identity --query Account --output text) - - # Verify and/or create ECR repositories - repos=("flotorch-app" "flotorch-indexing" "flotorch-retriever" "flotorch-evaluation" "flotorch-runtime" "flotorch-costcompute") - - for repo in "${repos[@]}"; do - repo_name="${repo}-${ENV_SUFFIX}" - if ! aws ecr describe-repositories --repository-names "$repo_name" --region ${AWS_REGION} >/dev/null 2>&1; then - echo "Creating repository: $repo_name" - aws ecr create-repository --repository-name "$repo_name" --region ${AWS_REGION} --image-scanning-configuration scanOnPush=true - else - echo "Repository $repo_name already exists" - fi - done - - # Build and push Docker images - echo "Building and pushing Docker images with suffix: ${ENV_SUFFIX}" - - # Build app image - echo "Building app image..." - docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-app-${ENV_SUFFIX}:latest -f app/Dockerfile . - docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-app-${ENV_SUFFIX}:latest - - # Build indexing image - echo "Building indexing image..." - docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-indexing-${ENV_SUFFIX}:latest -f indexing/fargate_indexing.Dockerfile . - docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-indexing-${ENV_SUFFIX}:latest - - # Build retriever image - echo "Building retriever image..." - docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-retriever-${ENV_SUFFIX}:latest -f retriever/fargate_retriever.Dockerfile . - docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-retriever-${ENV_SUFFIX}:latest - - # Build evaluation image - echo "Building evaluation image..." - docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-evaluation-${ENV_SUFFIX}:latest -f evaluation/fargate_evaluation.Dockerfile . - docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-evaluation-${ENV_SUFFIX}:latest - - # Build runtime image - echo "Building runtime image..." - docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-runtime-${ENV_SUFFIX}:latest -f opensearch/opensearch.Dockerfile . - docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-runtime-${ENV_SUFFIX}:latest - - # Build cost compute image - echo "Building cost compute image..." - cd lambda_handlers - docker build --platform linux/amd64 -t ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-costcompute-${ENV_SUFFIX}:latest -f cost_handler/Dockerfile . - docker push ${account_id}.dkr.ecr.${AWS_REGION}.amazonaws.com/flotorch-costcompute-${ENV_SUFFIX}:latest - cd .. - ''' + script { + def services = [ + [name: 'app', dockerfile: 'app/Dockerfile'], + [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], + [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], + [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], + [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] + ] + + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + try { + // ECR Login + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ + + // Build and push each service + services.each { service -> + def repoName = "flotorch-${service.name}-${ENV_SUFFIX}" + def imageTag = "${ECR_REGISTRY}/${repoName}:latest" + echo "Building and pushing ${service.name} image..." + + // Check if Dockerfile exists + if (!fileExists(service.dockerfile)) { + error "Dockerfile not found at ${service.dockerfile}" + } + + echo "Using Dockerfile at: ${service.dockerfile}" + + // Create ECR repository if it doesn't exist + sh """ + if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then + echo "Creating ECR repository: ${repoName}" + aws ecr create-repository --repository-name ${repoName} --region ${AWS_REGION} + fi + """ + + // Build and push + try { + sh """ + # Build image with linux/amd64 platform + docker build --platform linux/amd64 -t ${imageTag} -f ${service.dockerfile} . || exit 1 + + # Push image + docker push ${imageTag} || exit 1 + """ + echo "${service.name} image built and pushed successfully" + } catch (Exception e) { + echo "Failed to build/push ${service.name} image: ${e.getMessage()}" + throw e + } + } + + // Build and push cost compute image separately + def costComputeRepo = "flotorch-costcompute-${ENV_SUFFIX}" + def costComputeTag = "${ECR_REGISTRY}/${costComputeRepo}:latest" + + sh """ + if ! aws ecr describe-repositories --repository-names ${costComputeRepo} --region ${AWS_REGION} 2>/dev/null; then + echo "Creating ECR repository: ${costComputeRepo}" + aws ecr create-repository --repository-name ${costComputeRepo} --region ${AWS_REGION} + fi + + cd lambda_handlers + docker build --platform linux/amd64 -t ${costComputeTag} -f cost_handler/Dockerfile . || exit 1 + docker push ${costComputeTag} || exit 1 + cd .. + """ + echo "Cost compute image built and pushed successfully" + + } catch (Exception e) { + echo "Error during build/push: ${e.getMessage()}" + throw e + } + } } } } stage('Display Build Summary') { steps { - withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { - sh ''' - # Get AWS account ID - account_id=$(aws sts get-caller-identity --query Account --output text) - - echo "===================================================" - echo "BUILD AND PUSH SUMMARY FOR ${ENV_SUFFIX}" - echo "===================================================" - echo "Environment Suffix: ${ENV_SUFFIX}" - echo "AWS Region: ${AWS_REGION}" - echo "ECR Account ID: ${account_id}" - echo "" - echo "Images Pushed:" - echo "- flotorch-app-${ENV_SUFFIX}:latest" - echo "- flotorch-indexing-${ENV_SUFFIX}:latest" - echo "- flotorch-retriever-${ENV_SUFFIX}:latest" - echo "- flotorch-evaluation-${ENV_SUFFIX}:latest" - echo "- flotorch-runtime-${ENV_SUFFIX}:latest" - echo "- flotorch-costcompute-${ENV_SUFFIX}:latest" - echo "" - echo "Repository Pattern: flotorch-{service}-${ENV_SUFFIX}" - echo "===================================================" - ''' + script { + echo """ + ================================================= + BUILD AND PUSH SUMMARY FOR ${ENV_SUFFIX} + ================================================= + Environment Suffix: ${ENV_SUFFIX} + AWS Region: ${AWS_REGION} + ECR Registry: ${ECR_REGISTRY} + + Images Pushed: + - flotorch-app-${ENV_SUFFIX}:latest + - flotorch-indexing-${ENV_SUFFIX}:latest + - flotorch-retriever-${ENV_SUFFIX}:latest + - flotorch-evaluation-${ENV_SUFFIX}:latest + - flotorch-runtime-${ENV_SUFFIX}:latest + - flotorch-costcompute-${ENV_SUFFIX}:latest + + Repository Pattern: flotorch-{service}-${ENV_SUFFIX} + ================================================= + """ } } } From 844cf83c86c48d829a119b316e803be67339dff6 Mon Sep 17 00:00:00 2001 From: shivani-fission Date: Thu, 27 Feb 2025 11:05:04 +0530 Subject: [PATCH 49/49] Update Jenkinsfile.imagepush --- Jenkins/Jenkinsfile.imagepush | 116 ++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 28 deletions(-) diff --git a/Jenkins/Jenkinsfile.imagepush b/Jenkins/Jenkinsfile.imagepush index 2ad5cff6..0b7cda15 100644 --- a/Jenkins/Jenkinsfile.imagepush +++ b/Jenkins/Jenkinsfile.imagepush @@ -29,7 +29,7 @@ pipeline { steps { sh ''' apt-get update - apt-get install -y docker.io curl unzip git + apt-get install -y docker.io curl unzip git jq curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip -o -q awscliv2.zip ./aws/install --update @@ -58,17 +58,22 @@ pipeline { withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { try { - // ECR Login - sh """ + // ECR Login - Added verification for successful login + def loginStatus = sh(script: """ aws ecr get-login-password --region ${AWS_REGION} | \ docker login --username AWS --password-stdin ${ECR_REGISTRY} - """ + """, returnStatus: true) + + if (loginStatus != 0) { + error "Failed to login to ECR. Please check your credentials." + } // Build and push each service services.each { service -> def repoName = "flotorch-${service.name}-${ENV_SUFFIX}" def imageTag = "${ECR_REGISTRY}/${repoName}:latest" - echo "Building and pushing ${service.name} image..." + + echo "Building and pushing ${service.name} image with tag latest..." // Check if Dockerfile exists if (!fileExists(service.dockerfile)) { @@ -78,45 +83,100 @@ pipeline { echo "Using Dockerfile at: ${service.dockerfile}" // Create ECR repository if it doesn't exist - sh """ + def repoCreationStatus = sh(script: """ if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then echo "Creating ECR repository: ${repoName}" aws ecr create-repository --repository-name ${repoName} --region ${AWS_REGION} + else + echo "Repository ${repoName} already exists" fi - """ + """, returnStatus: true) - // Build and push - try { - sh """ - # Build image with linux/amd64 platform - docker build --platform linux/amd64 -t ${imageTag} -f ${service.dockerfile} . || exit 1 - - # Push image - docker push ${imageTag} || exit 1 - """ - echo "${service.name} image built and pushed successfully" - } catch (Exception e) { - echo "Failed to build/push ${service.name} image: ${e.getMessage()}" - throw e + if (repoCreationStatus != 0) { + error "Failed to create or verify ECR repository: ${repoName}" } + + // Build image with no cache to ensure fresh build + def buildStatus = sh(script: """ + # Build image with linux/amd64 platform and no cache + docker build --no-cache --platform linux/amd64 -t ${imageTag} -f ${service.dockerfile} . + """, returnStatus: true) + + if (buildStatus != 0) { + error "Failed to build image: ${imageTag}" + } + + // Push the image + def pushStatus = sh(script: """ + # Push image with latest tag + docker push ${imageTag} + """, returnStatus: true) + + if (pushStatus != 0) { + error "Failed to push image: ${imageTag}" + } + + // Verify the push was successful + def verifyStatus = sh(script: """ + # Verify the image exists in ECR + aws ecr describe-images --repository-name ${repoName} --image-ids imageTag=latest --region ${AWS_REGION} + """, returnStatus: true) + + if (verifyStatus != 0) { + error "Failed to verify image in ECR: ${repoName}:latest" + } + + echo "${service.name} image built and pushed successfully with tag: latest" } // Build and push cost compute image separately def costComputeRepo = "flotorch-costcompute-${ENV_SUFFIX}" - def costComputeTag = "${ECR_REGISTRY}/${costComputeRepo}:latest" + def costImageTag = "${ECR_REGISTRY}/${costComputeRepo}:latest" - sh """ + def costRepoStatus = sh(script: """ if ! aws ecr describe-repositories --repository-names ${costComputeRepo} --region ${AWS_REGION} 2>/dev/null; then echo "Creating ECR repository: ${costComputeRepo}" aws ecr create-repository --repository-name ${costComputeRepo} --region ${AWS_REGION} + else + echo "Repository ${costComputeRepo} already exists" fi - + """, returnStatus: true) + + if (costRepoStatus != 0) { + error "Failed to create or verify ECR repository: ${costComputeRepo}" + } + + def costBuildStatus = sh(script: """ cd lambda_handlers - docker build --platform linux/amd64 -t ${costComputeTag} -f cost_handler/Dockerfile . || exit 1 - docker push ${costComputeTag} || exit 1 - cd .. - """ - echo "Cost compute image built and pushed successfully" + # Build image with no cache + docker build --no-cache --platform linux/amd64 -t ${costImageTag} -f cost_handler/Dockerfile . + """, returnStatus: true) + + if (costBuildStatus != 0) { + error "Failed to build cost compute image: ${costImageTag}" + } + + // Push the cost compute image + def costPushStatus = sh(script: """ + # Push image with latest tag + docker push ${costImageTag} + """, returnStatus: true) + + if (costPushStatus != 0) { + error "Failed to push cost compute image: ${costImageTag}" + } + + // Verify the push was successful + def costVerifyStatus = sh(script: """ + # Verify the image exists in ECR + aws ecr describe-images --repository-name ${costComputeRepo} --image-ids imageTag=latest --region ${AWS_REGION} + """, returnStatus: true) + + if (costVerifyStatus != 0) { + error "Failed to verify cost compute image in ECR: ${costComputeRepo}:latest" + } + + echo "Cost compute image built and pushed successfully with tag: latest" } catch (Exception e) { echo "Error during build/push: ${e.getMessage()}"