diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 01cf5083..00000000 --- a/Dockerfile +++ /dev/null @@ -1,126 +0,0 @@ -# Use Amazon Linux 2023 as base image -FROM amazonlinux:2023 - -# Install system dependencies in stages to handle package conflicts properly -RUN dnf update -y && \ - dnf clean all - -# Install basic build tools first -RUN dnf install -y \ - gcc-c++ \ - make \ - shadow-utils \ - openssl \ - openssl-devel \ - && dnf clean all - -# Install Python and development tools -RUN dnf install -y \ - python3 \ - python3-pip \ - git \ - && dnf clean all - -# Install compression tools -RUN dnf install -y \ - tar \ - gzip \ - unzip \ - && dnf clean all - -# Install jq -RUN dnf install -y \ - jq && \ - dnf clean all - -# Install Docker and required dependencies -RUN dnf install -y docker iptables procps && \ - dnf clean all - -# Create necessary directories with proper permissions -RUN mkdir -p /run/docker && \ - mkdir -p /var/lib/docker && \ - mkdir -p /etc/docker - -# Configure Docker daemon -RUN echo '{"storage-driver": "vfs"}' > /etc/docker/daemon.json - -# Install Node.js and npm using nvm -ENV NVM_DIR=/root/.nvm -ENV NODE_VERSION=22.12.0 - -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \ - . $NVM_DIR/nvm.sh && \ - nvm install $NODE_VERSION && \ - nvm alias default $NODE_VERSION && \ - nvm use default - -# Add node and npm to path so the commands are available -ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH - -# Verify Node.js and npm versions -RUN node -v && npm -v - -# Install AWS CLI v2 -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && \ - ./aws/install && \ - rm -rf aws awscliv2.zip - -# Install AWS CDK globally -RUN npm install -g aws-cdk && \ - cdk --version - -# Set working directory -WORKDIR /FloTorch - -# Copy the entire FloTorch directory -COPY . . - -# Make deploy.sh executable -RUN chmod +x /FloTorch/cdk/deploy.sh - -# Install required Python packages -RUN pip3 install -r cdk/requirements.txt --upgrade - -# Create the entrypoint script -RUN echo '#!/bin/bash' > /entrypoint.sh && \ - echo 'set -e' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Start Docker daemon if not running' >> /entrypoint.sh && \ - echo 'if ! pgrep dockerd >/dev/null 2>&1; then' >> /entrypoint.sh && \ - echo ' echo "Starting Docker daemon..."' >> /entrypoint.sh && \ - echo ' dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --storage-driver=vfs &' >> /entrypoint.sh && \ - echo ' echo "Waiting for Docker daemon to start..."' >> /entrypoint.sh && \ - echo ' for i in $(seq 1 30); do' >> /entrypoint.sh && \ - echo ' if docker info >/dev/null 2>&1; then' >> /entrypoint.sh && \ - echo ' echo "Docker daemon started successfully"' >> /entrypoint.sh && \ - echo ' break' >> /entrypoint.sh && \ - echo ' fi' >> /entrypoint.sh && \ - echo ' echo "Waiting for Docker daemon... ($i/30)"' >> /entrypoint.sh && \ - echo ' sleep 1' >> /entrypoint.sh && \ - echo ' done' >> /entrypoint.sh && \ - echo 'fi' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Print environment for debugging' >> /entrypoint.sh && \ - echo 'echo "AWS credentials setup..."' >> /entrypoint.sh && \ - echo 'aws configure list' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Verify required environment variables' >> /entrypoint.sh && \ - echo 'if [ -z "${AWS_ACCOUNT_ID}" ] || [ -z "${AWS_DEFAULT_REGION}" ]; then' >> /entrypoint.sh && \ - echo ' echo "Error: AWS_ACCOUNT_ID and AWS_DEFAULT_REGION must be provided"' >> /entrypoint.sh && \ - echo ' exit 1' >> /entrypoint.sh && \ - echo 'fi' >> /entrypoint.sh && \ - echo '' >> /entrypoint.sh && \ - echo '# Run deployment with verbose output' >> /entrypoint.sh && \ - echo 'cd /FloTorch/cdk' >> /entrypoint.sh && \ - echo 'echo "Starting deployment process..."' >> /entrypoint.sh && \ - echo 'echo "Using AWS Account: ${AWS_ACCOUNT_ID}"' >> /entrypoint.sh && \ - echo 'echo "Using AWS Region: ${AWS_DEFAULT_REGION}"' >> /entrypoint.sh && \ - echo './deploy.sh' >> /entrypoint.sh && \ - echo 'echo "Deployment completed."' >> /entrypoint.sh && \ - chmod +x /entrypoint.sh - -# Set the entrypoint -ENTRYPOINT ["/entrypoint.sh"] diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile new file mode 100644 index 00000000..2e752e98 --- /dev/null +++ b/Jenkins/Jenkinsfile @@ -0,0 +1,114 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + + parameters { + string(name: 'AWS_REGION', defaultValue: 'us-east-1', description: 'AWS Region (e.g., us-east-1)') + string(name: 'PROJECT_NAME', defaultValue: 'flotorch', description: 'Project Name (e.g., flotorch)') + string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') + string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') + string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') + string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1)') + string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') + password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') + password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') + } + + environment { + AWS_REGION = "${params.AWS_REGION}" + PROJECT_NAME = "${params.PROJECT_NAME}" + TABLE_SUFFIX = "${params.TABLE_SUFFIX}" + CLIENT_NAME = "${params.CLIENT_NAME}" + CREATED_BY = "${params.CREATED_BY}" + TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" + OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" + OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" + NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" + } + + stages { + stage('Install dependencies') { + steps { + script { + sh ''' + echo "Installing AWS CLI..." + apt-get update + apt-get install -y unzip curl + + # Check if AWS CLI is already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + else + echo "AWS CLI is already installed" + aws --version + fi + + # Verify AWS CLI installation + aws --version || { + echo "AWS CLI installation failed" + exit 1 + } + echo "AWS CLI installation successful" + ''' + } + } + } + + stage('Checkout Repository') { + steps { + script { + git branch: 'main', url: 'https://github.com/FissionAI/FloTorch.git' + } + } + } + + stage('Deploy FloTorch Master Stack') { + steps { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh ''' + echo "Starting FloTorch Stack Deployment..." + + aws cloudformation create-stack \ + --region ${AWS_REGION} \ + --stack-name flotorch-stack \ + --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ + --parameters \ + ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ + ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ + ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ + ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ + ParameterKey=TemplateVersion,ParameterValue=${TEMPLATE_VERSION} \ + ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ + ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ + ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} \ + ParameterKey=PrerequisitesMet,ParameterValue=yes + + echo "Waiting for stack creation to complete..." + aws cloudformation wait stack-create-complete \ + --stack-name flotorch-stack \ + --region ${AWS_REGION} + + echo "CloudFormation Stack Deployment Successful!" + ''' + } + } + } + } + + post { + success { + echo "FloTorch Stack deployed successfully!" + } + failure { + echo "Deployment failed!" + } + } +} diff --git a/Jenkins/Jenkinsfile.deployment b/Jenkins/Jenkinsfile.deployment new file mode 100644 index 00000000..8860dbfe --- /dev/null +++ b/Jenkins/Jenkinsfile.deployment @@ -0,0 +1,250 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + parameters { + choice(name: 'ENVIRONMENT', choices: ['dev', 'qa', 'prod'], description: 'Select deployment environment (dev/qa/prod)') + string(name: 'BUILD_VERSION', defaultValue: 'latest', description: 'Specify build version') + booleanParam(name: 'FORCE_REBUILD', defaultValue: true, description: 'Force rebuild even if image with the same tag exists') + } + environment { + ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' + AWS_REGION = 'us-east-1' + ENV_NAME = "${params.ENVIRONMENT == 'dev' ? 'dampen' : params.ENVIRONMENT == 'qa' ? 'iamdqa' : 'prod'}" + } + stages { + stage('Install Dependencies') { + steps { + script { + sh ''' + apt-get update + apt-get install -y docker.io curl unzip git jq + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + ''' + } + } + } + stage('Checkout') { + steps { + cleanWs() + git branch: 'main', + url: 'https://github.com/shivani-fission/FloTorch.git' + } + } + stage('Clean Up Old Tags') { + steps { + script { + // Define services + def services = ['app', 'indexing', 'retriever', 'evaluation'] + + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + services.each { service -> + def repoName = "flotorch-${service}-${ENV_NAME}" + + echo "Cleaning up repository: ${repoName}" + + try { + // Check if repository exists + sh """ + if aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then + echo "Repository exists: ${repoName}" + + # Only remove comma-separated multi-tagged images + MULTI_TAGS=\$(aws ecr list-images --repository-name ${repoName} --query "imageIds[?contains(imageTag, ',')].imageTag" --output json) + if [ "\$(echo \$MULTI_TAGS | jq 'length')" -gt "0" ]; then + echo "Found multi-tagged images, removing them..." + echo \$MULTI_TAGS | jq -r '.[]' | while read -r tag; do + aws ecr batch-delete-image --repository-name ${repoName} --image-ids imageTag="\$tag" || true + done + fi + + # Only if FORCE_REBUILD is true, remove the specific version tag if it exists + if [ "${params.FORCE_REBUILD}" = "true" ]; then + echo "Checking if version ${params.BUILD_VERSION} exists..." + if aws ecr describe-images --repository-name ${repoName} --image-ids imageTag=${params.BUILD_VERSION} 2>/dev/null; then + echo "Removing existing image with tag ${params.BUILD_VERSION}" + aws ecr batch-delete-image --repository-name ${repoName} --image-ids imageTag=${params.BUILD_VERSION} || true + fi + fi + else + echo "Repository does not exist: ${repoName}" + fi + """ + } catch (Exception e) { + echo "Warning: Error during cleanup for ${repoName}: ${e.getMessage()}" + // Continue with next repository + } + } + } + } + } + } + stage('Build and Push Images') { + steps { + script { + // Always build all four services + def services = ['app', 'indexing', 'retriever', 'evaluation'] + + def builtImages = [] + def failedImages = [] + + // Display environment-specific header in console + echo """ + =================================================== + BUILDING IMAGES FOR: ${params.ENVIRONMENT.toUpperCase()} + BUILD VERSION: ${params.BUILD_VERSION} + ALL SERVICES: ${services.join(', ')} + =================================================== + """ + + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + try { + // ECR Login + sh """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """ + + // Build and push each service + services.each { service -> + def repoName = "flotorch-${service}-${ENV_NAME}" + def imageTagWithVersion = "${ECR_REGISTRY}/${repoName}:${params.BUILD_VERSION}" + def imageTagLatest = "${ECR_REGISTRY}/${repoName}:latest" + + echo "Building and pushing ${service} image for ${params.ENVIRONMENT}..." + + // Get the correct Dockerfile path and verify it exists + def dockerfilePath = "${service}/Dockerfile" + + // Check if Dockerfile exists + if (!fileExists(dockerfilePath)) { + echo "WARNING: Dockerfile not found at ${dockerfilePath}, creating empty file for testing" + // Create an empty Dockerfile for testing purposes + sh "mkdir -p ${service}" + sh """ + cat > ${dockerfilePath} << 'EOL' +FROM ubuntu:latest +RUN echo "This is a test image for ${service} - built on \$(date)" +CMD ["echo", "Hello from ${service}"] +EOL + """ + } + + echo "Using Dockerfile at: ${dockerfilePath}" + + // Create ECR repository if it doesn't exist + sh """ + if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then + echo "Creating ECR repository: ${repoName}" + aws ecr create-repository --repository-name ${repoName} --region ${AWS_REGION} + fi + """ + + // Build and push + try { + // Add a timestamp to the image to ensure it's unique + def timestamp = new Date().format("yyyyMMdd-HHmmss") + def uniqueImageTag = "${ECR_REGISTRY}/${repoName}:build-${timestamp}" + + sh """ + # Build image with a unique temporary tag first to avoid reusing existing images + echo "Building fresh image with timestamp ${timestamp}" + docker build --no-cache --build-arg BUILD_DATE="${timestamp}" -t ${uniqueImageTag} -f ${dockerfilePath} . || exit 1 + + # Tag with the requested version + docker tag ${uniqueImageTag} ${imageTagWithVersion} + + # Push the version-tagged image + docker push ${imageTagWithVersion} || exit 1 + + echo "Tagged and pushed as ${params.BUILD_VERSION} only" + """ + + builtImages.add([ + service: service, + repo: repoName, + image: imageTagWithVersion, + environment: params.ENVIRONMENT + ]) + echo "${service} image built and pushed to ${params.ENVIRONMENT} successfully" + } catch (Exception e) { + echo "Failed to build/push ${service} image for ${params.ENVIRONMENT}: ${e.getMessage()}" + failedImages.add(service) + } + } + + // Check if any images failed + if (failedImages.size() > 0) { + error "Failed to build/push the following images: ${failedImages.join(', ')}" + } + + // Create dashboard summary + def summary = "Images pushed to ${params.ENVIRONMENT.toUpperCase()} repositories:\n" + builtImages.each { img -> + summary += "- ${img.service}: ${img.image}\n" + } + + // Display dashboard summary + echo summary + + // Create a build artifact with the summary + writeFile file: "${params.ENVIRONMENT}-images.txt", text: summary + archiveArtifacts artifacts: "${params.ENVIRONMENT}-images.txt", allowEmptyArchive: true + + // Add environment information to build description + currentBuild.description = "ENV: ${params.ENVIRONMENT.toUpperCase()} | Version: ${params.BUILD_VERSION} | Services: ${builtImages.size()}/4" + + } catch (Exception e) { + echo "Error during build/push for ${params.ENVIRONMENT}: ${e.getMessage()}" + throw e + } + } + } + } + } + stage('Display Environment Dashboard') { + steps { + script { + echo """ + ================================================= + DEPLOYMENT SUMMARY FOR ${params.ENVIRONMENT.toUpperCase()} + ================================================= + Environment: ${params.ENVIRONMENT} + Env Name: ${ENV_NAME} + Build Version: ${params.BUILD_VERSION} + Registry: ${ECR_REGISTRY} + + Services Deployed: + - app: ${ECR_REGISTRY}/flotorch-app-${ENV_NAME}:${params.BUILD_VERSION} + - indexing: ${ECR_REGISTRY}/flotorch-indexing-${ENV_NAME}:${params.BUILD_VERSION} + - retriever: ${ECR_REGISTRY}/flotorch-retriever-${ENV_NAME}:${params.BUILD_VERSION} + - evaluation: ${ECR_REGISTRY}/flotorch-evaluation-${ENV_NAME}:${params.BUILD_VERSION} + + Repository Pattern: flotorch-{service}-${ENV_NAME} + ================================================= + """ + + // Set the build display name to show environment + currentBuild.displayName = "#${BUILD_NUMBER} - ${params.ENVIRONMENT.toUpperCase()}" + } + } + } + } + post { + success { + echo "Successfully built and pushed all images for ${params.ENVIRONMENT} with version ${params.BUILD_VERSION}" + } + failure { + echo "Failed to build/push images for ${params.ENVIRONMENT}" + } + always { + cleanWs() + sh 'docker system prune -f' + } + } +} diff --git a/Jenkins/Jenkinsfile.imagepush b/Jenkins/Jenkinsfile.imagepush new file mode 100644 index 00000000..0b7cda15 --- /dev/null +++ b/Jenkins/Jenkinsfile.imagepush @@ -0,0 +1,229 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + parameters { + string(name: 'ENVIRONMENT_SUFFIX', defaultValue: '', description: 'Enter the environment suffix for ECR repositories (e.g., jasper)') + } + environment { + ECR_REGISTRY = '677276078734.dkr.ecr.us-east-1.amazonaws.com' + AWS_REGION = 'us-east-1' + ENV_SUFFIX = "${params.ENVIRONMENT_SUFFIX}" + } + stages { + stage('Validate Parameters') { + steps { + script { + if (params.ENVIRONMENT_SUFFIX.trim() == '') { + error "ENVIRONMENT_SUFFIX parameter is required. Please provide a valid value." + } + echo "Building and pushing images with suffix: ${ENV_SUFFIX}" + } + } + } + + stage('Install Dependencies') { + steps { + sh ''' + apt-get update + apt-get install -y docker.io curl unzip git jq + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + ''' + } + } + + stage('Checkout') { + steps { + cleanWs() + git branch: 'main', + url: 'https://github.com/shivani-fission/FloTorch.git' + } + } + + stage('Build and Push Images') { + steps { + script { + def services = [ + [name: 'app', dockerfile: 'app/Dockerfile'], + [name: 'indexing', dockerfile: 'indexing/fargate_indexing.Dockerfile'], + [name: 'retriever', dockerfile: 'retriever/fargate_retriever.Dockerfile'], + [name: 'evaluation', dockerfile: 'evaluation/fargate_evaluation.Dockerfile'], + [name: 'runtime', dockerfile: 'opensearch/opensearch.Dockerfile'] + ] + + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + try { + // ECR Login - Added verification for successful login + def loginStatus = sh(script: """ + aws ecr get-login-password --region ${AWS_REGION} | \ + docker login --username AWS --password-stdin ${ECR_REGISTRY} + """, returnStatus: true) + + if (loginStatus != 0) { + error "Failed to login to ECR. Please check your credentials." + } + + // Build and push each service + services.each { service -> + def repoName = "flotorch-${service.name}-${ENV_SUFFIX}" + def imageTag = "${ECR_REGISTRY}/${repoName}:latest" + + echo "Building and pushing ${service.name} image with tag latest..." + + // Check if Dockerfile exists + if (!fileExists(service.dockerfile)) { + error "Dockerfile not found at ${service.dockerfile}" + } + + echo "Using Dockerfile at: ${service.dockerfile}" + + // Create ECR repository if it doesn't exist + def repoCreationStatus = sh(script: """ + if ! aws ecr describe-repositories --repository-names ${repoName} --region ${AWS_REGION} 2>/dev/null; then + echo "Creating ECR repository: ${repoName}" + aws ecr create-repository --repository-name ${repoName} --region ${AWS_REGION} + else + echo "Repository ${repoName} already exists" + fi + """, returnStatus: true) + + if (repoCreationStatus != 0) { + error "Failed to create or verify ECR repository: ${repoName}" + } + + // Build image with no cache to ensure fresh build + def buildStatus = sh(script: """ + # Build image with linux/amd64 platform and no cache + docker build --no-cache --platform linux/amd64 -t ${imageTag} -f ${service.dockerfile} . + """, returnStatus: true) + + if (buildStatus != 0) { + error "Failed to build image: ${imageTag}" + } + + // Push the image + def pushStatus = sh(script: """ + # Push image with latest tag + docker push ${imageTag} + """, returnStatus: true) + + if (pushStatus != 0) { + error "Failed to push image: ${imageTag}" + } + + // Verify the push was successful + def verifyStatus = sh(script: """ + # Verify the image exists in ECR + aws ecr describe-images --repository-name ${repoName} --image-ids imageTag=latest --region ${AWS_REGION} + """, returnStatus: true) + + if (verifyStatus != 0) { + error "Failed to verify image in ECR: ${repoName}:latest" + } + + echo "${service.name} image built and pushed successfully with tag: latest" + } + + // Build and push cost compute image separately + def costComputeRepo = "flotorch-costcompute-${ENV_SUFFIX}" + def costImageTag = "${ECR_REGISTRY}/${costComputeRepo}:latest" + + def costRepoStatus = sh(script: """ + if ! aws ecr describe-repositories --repository-names ${costComputeRepo} --region ${AWS_REGION} 2>/dev/null; then + echo "Creating ECR repository: ${costComputeRepo}" + aws ecr create-repository --repository-name ${costComputeRepo} --region ${AWS_REGION} + else + echo "Repository ${costComputeRepo} already exists" + fi + """, returnStatus: true) + + if (costRepoStatus != 0) { + error "Failed to create or verify ECR repository: ${costComputeRepo}" + } + + def costBuildStatus = sh(script: """ + cd lambda_handlers + # Build image with no cache + docker build --no-cache --platform linux/amd64 -t ${costImageTag} -f cost_handler/Dockerfile . + """, returnStatus: true) + + if (costBuildStatus != 0) { + error "Failed to build cost compute image: ${costImageTag}" + } + + // Push the cost compute image + def costPushStatus = sh(script: """ + # Push image with latest tag + docker push ${costImageTag} + """, returnStatus: true) + + if (costPushStatus != 0) { + error "Failed to push cost compute image: ${costImageTag}" + } + + // Verify the push was successful + def costVerifyStatus = sh(script: """ + # Verify the image exists in ECR + aws ecr describe-images --repository-name ${costComputeRepo} --image-ids imageTag=latest --region ${AWS_REGION} + """, returnStatus: true) + + if (costVerifyStatus != 0) { + error "Failed to verify cost compute image in ECR: ${costComputeRepo}:latest" + } + + echo "Cost compute image built and pushed successfully with tag: latest" + + } catch (Exception e) { + echo "Error during build/push: ${e.getMessage()}" + throw e + } + } + } + } + } + + stage('Display Build Summary') { + steps { + script { + echo """ + ================================================= + BUILD AND PUSH SUMMARY FOR ${ENV_SUFFIX} + ================================================= + Environment Suffix: ${ENV_SUFFIX} + AWS Region: ${AWS_REGION} + ECR Registry: ${ECR_REGISTRY} + + Images Pushed: + - flotorch-app-${ENV_SUFFIX}:latest + - flotorch-indexing-${ENV_SUFFIX}:latest + - flotorch-retriever-${ENV_SUFFIX}:latest + - flotorch-evaluation-${ENV_SUFFIX}:latest + - flotorch-runtime-${ENV_SUFFIX}:latest + - flotorch-costcompute-${ENV_SUFFIX}:latest + + Repository Pattern: flotorch-{service}-${ENV_SUFFIX} + ================================================= + """ + } + } + } + } + + post { + success { + echo "All images successfully built and pushed to ECR repositories with suffix: ${ENV_SUFFIX}" + } + failure { + echo "Failed to build and push images to ECR repositories" + } + always { + cleanWs() + sh 'docker system prune -f' + } + } +} diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..2e752e98 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,114 @@ +pipeline { + agent { + docker { + image 'ubuntu:latest' + args '-u root' + } + } + + parameters { + string(name: 'AWS_REGION', defaultValue: 'us-east-1', description: 'AWS Region (e.g., us-east-1)') + string(name: 'PROJECT_NAME', defaultValue: 'flotorch', description: 'Project Name (e.g., flotorch)') + string(name: 'TABLE_SUFFIX', defaultValue: '', description: 'Unique Table Suffix (6 lowercase letters)') + string(name: 'CLIENT_NAME', defaultValue: 'flotorch', description: 'Client Name (e.g., flotorch)') + string(name: 'CREATED_BY', defaultValue: 'DevOpsTeam', description: 'Created by (e.g., DevOpsTeam)') + string(name: 'TEMPLATE_VERSION', defaultValue: 'v1.0.0', description: 'Template Version (e.g., 2.0.1)') + string(name: 'OPENSEARCH_ADMIN_USER', defaultValue: 'admin', description: 'OpenSearch Admin Username') + password(name: 'OPENSEARCH_ADMIN_PASSWORD', description: 'OpenSearch Admin Password') + password(name: 'NGINX_AUTH_PASSWORD', description: 'Nginx Auth Password') + } + + environment { + AWS_REGION = "${params.AWS_REGION}" + PROJECT_NAME = "${params.PROJECT_NAME}" + TABLE_SUFFIX = "${params.TABLE_SUFFIX}" + CLIENT_NAME = "${params.CLIENT_NAME}" + CREATED_BY = "${params.CREATED_BY}" + TEMPLATE_VERSION = "${params.TEMPLATE_VERSION}" + OPENSEARCH_ADMIN_USER = "${params.OPENSEARCH_ADMIN_USER}" + OPENSEARCH_ADMIN_PASSWORD = "${params.OPENSEARCH_ADMIN_PASSWORD}" + NGINX_AUTH_PASSWORD = "${params.NGINX_AUTH_PASSWORD}" + } + + stages { + stage('Install dependencies') { + steps { + script { + sh ''' + echo "Installing AWS CLI..." + apt-get update + apt-get install -y unzip curl + + # Check if AWS CLI is already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o -q awscliv2.zip + ./aws/install --update + else + echo "AWS CLI is already installed" + aws --version + fi + + # Verify AWS CLI installation + aws --version || { + echo "AWS CLI installation failed" + exit 1 + } + echo "AWS CLI installation successful" + ''' + } + } + } + + stage('Checkout Repository') { + steps { + script { + git branch: 'main', url: 'https://github.com/FissionAI/FloTorch.git' + } + } + } + + stage('Deploy FloTorch Master Stack') { + steps { + withAWS(credentials: 'aws-creds', region: env.AWS_REGION) { + sh ''' + echo "Starting FloTorch Stack Deployment..." + + aws cloudformation create-stack \ + --region ${AWS_REGION} \ + --stack-name flotorch-stack \ + --template-url https://flotorch-public.s3.amazonaws.com/${TEMPLATE_VERSION}/templates/master-template.yaml \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ + --parameters \ + ParameterKey=ProjectName,ParameterValue=${PROJECT_NAME} \ + ParameterKey=TableSuffix,ParameterValue=${TABLE_SUFFIX} \ + ParameterKey=ClientName,ParameterValue=${CLIENT_NAME} \ + ParameterKey=CreatedBy,ParameterValue=${CREATED_BY} \ + ParameterKey=TemplateVersion,ParameterValue=${TEMPLATE_VERSION} \ + ParameterKey=OpenSearchAdminUser,ParameterValue=${OPENSEARCH_ADMIN_USER} \ + ParameterKey=OpenSearchAdminPassword,ParameterValue=${OPENSEARCH_ADMIN_PASSWORD} \ + ParameterKey=NginxAuthPassword,ParameterValue=${NGINX_AUTH_PASSWORD} \ + ParameterKey=PrerequisitesMet,ParameterValue=yes + + echo "Waiting for stack creation to complete..." + aws cloudformation wait stack-create-complete \ + --stack-name flotorch-stack \ + --region ${AWS_REGION} + + echo "CloudFormation Stack Deployment Successful!" + ''' + } + } + } + } + + post { + success { + echo "FloTorch Stack deployed successfully!" + } + failure { + echo "Deployment failed!" + } + } +} diff --git a/app/Dockerfile b/app/Dockerfile index 02d5b749..f8066db5 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -22,7 +22,6 @@ RUN pnpm install && pnpm run generate FROM base AS release WORKDIR /app - # Install required packages RUN apt-get update && apt-get install -y \ nginx \ @@ -39,51 +38,9 @@ COPY app/nginx /etc/nginx # Create directory for supervisor configs RUN mkdir -p /etc/supervisor/conf.d -# Create entrypoint script -COPY <