diff --git a/.github/ISSUE_TEMPLATE/release-request.yml b/.github/ISSUE_TEMPLATE/release-request.yml new file mode 100644 index 0000000..2ed9717 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release-request.yml @@ -0,0 +1,53 @@ +name: "Release Request" +description: "Request a deployment by specifying the evaluation function, changes, target commit/branch, and test confirmation." +title: "Release Request" +labels: + - release + - deployment +assignees: [] +body: + - type: textarea + id: description_of_changes + attributes: + label: Description of changes + description: "Summarize what is changing and why. Include links to PRs, issues, or changelogs." + placeholder: | + - What changed: + - Why: + - Related PRs/Issues: #123, #456 + render: markdown + validations: + required: true + + - type: input + id: branch_to_deploy + attributes: + label: Branch to deploy + description: | + Specify Branch name to deploy. + placeholder: "e.g., release/2025-09-29" + validations: + required: true + + - type: dropdown + id: version-bump + attributes: + label: "๐Ÿš€ What kind of update is this?" + description: "Tell us how significant this change is. This helps us set the correct new version number." + options: + - "Patch: A small fix for a bug. It won't break anything for existing users. (e.g., 1.2.3 โž” 1.2.4)" + - "Minor: Adds a new feature, but doesn't change how existing ones work. A safe update. (e.g., 1.2.3 โž” 1.3.0)" + - "Major: A big change that alters existing features. Users may need to update their work to adapt. (e.g., 1.2.3 โž” 2.0.0)" + default: 0 + validations: + required: true + + - type: markdown + attributes: + value: | + --- + ### โšก Click the Link Below to Run the Workflow + + Clicking the link will take you to the Actions page. You will need to click the **"Run workflow"** button there to start the process. + + ## [โžก๏ธ Go to Workflow Run Page](../actions/workflows/production-deploy.yml) \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100755 index ec94d1e..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Build, Test and Deploy - -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -jobs: - test: - name: Test - runs-on: ubuntu-latest - permissions: - contents: read - actions: read - checks: write - pull-requests: write - strategy: - fail-fast: false - matrix: - python-version: ["3.12"] - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python ${{ matrix.python-version }} - id: python-setup - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Load cached Poetry installation - id: poetry-cache - uses: actions/cache@v4 - with: - path: ~/.local - key: poetry-0 - - - name: Install and configure Poetry - if: steps.poetry-cache.outputs.cache-hit != 'true' - uses: snok/install-poetry@v1 - with: - virtualenvs-in-project: true - - - name: Load cached venv - id: dependencies-cache - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.dependencies-cache.outputs.cache-hit != 'true' - run: | - poetry install --no-interaction --no-root - - # TODO: add linting / black / flake8 - # - name: Lint with flake8 - # run: | - # source .venv/bin/activate - # # stop the build if there are Python syntax errors or undefined names - # flake8 ./evaluation_function --count --select=E9,F63,F7,F82 --show-source --statistics - # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 ./evaluation_function --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Run tests - if: always() - run: | - source .venv/bin/activate - pytest --junit-xml=./reports/pytest.xml --tb=auto -v - - - name: Upload test results - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-results - path: ./reports/pytest.xml - if-no-files-found: warn - - build: - name: Build Docker Image - uses: lambda-feedback/evaluation-function-workflows/.github/workflows/build.yml@main - needs: test - permissions: - contents: read - id-token: write - packages: write - - deploy: - name: Deploy to Lambda Feedback - uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@main - needs: test - with: - template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" - permissions: - contents: read - id-token: write - packages: write - secrets: - aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} - aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} - function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}} diff --git a/.github/workflows/pre_production_tests.yml b/.github/workflows/pre_production_tests.yml new file mode 100644 index 0000000..2daabd1 --- /dev/null +++ b/.github/workflows/pre_production_tests.yml @@ -0,0 +1,49 @@ +name: Run Pre-Production Validation Tests + +on: + workflow_dispatch: + inputs: + eval_function: + type: string + description: "The name of the evaluation function to test" + required: true + sql_limit: + type: number + description: "The maximum number of SQL test cases to run" + required: false + default: 500 + seed: + type: string + description: "Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate." + required: false + default: '' + request_delay: + type: string + description: "Delay (seconds) between dispatching requests" + required: false + default: '0' + max_concurrency: + type: string + description: "Max concurrent requests (lower for GPT-backed functions)" + required: false + default: '5' + +jobs: + run-pre-production-tests: + name: ๐Ÿงช Run Staging Validation Tests + uses: lambda-feedback/Database-Testing/.github/workflows/test_evaluation_function.yml@main + with: + eval_function: ${{ inputs.eval_function }} + sql_limit: ${{ inputs.sql_limit }} + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + secrets: + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_DB_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml new file mode 100644 index 0000000..901c8d9 --- /dev/null +++ b/.github/workflows/production-deploy.yml @@ -0,0 +1,60 @@ +name: Deploy to Production + +on: + workflow_dispatch: + inputs: + version-bump: + description: 'Version bump type' + required: true + type: choice + options: + - patch + - minor + - major + default: patch + branch: + description: 'Branch to release from' + required: true + type: string + default: 'main' + seed: + description: 'Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate.' + required: false + type: string + default: '' + request_delay: + description: 'Delay (seconds) between dispatching requests' + required: false + type: string + default: '0' + max_concurrency: + description: 'Max concurrent requests (lower for GPT-backed functions)' + required: false + type: string + default: '5' +jobs: + deploy: + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@main + with: + template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-python' + environment: "production" + version-bump: ${{ inputs.version-bump }} + branch: ${{ inputs.branch }} + run-database-tests: false + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_DB_CREDS: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml new file mode 100644 index 0000000..c693871 --- /dev/null +++ b/.github/workflows/staging-deploy.yml @@ -0,0 +1,65 @@ +name: Deploy to Staging + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + run: pip install poetry + + - name: Install dependencies + run: poetry install + + - name: Lint with flake8 + run: | + poetry run flake8 ./evaluation_function --count --select=E9,F63,F7,F82 --show-source --statistics + poetry run flake8 ./evaluation_function --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run tests + if: always() + run: poetry run pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.python-version }} + path: ./reports/pytest.xml + if-no-files-found: warn + deploy: + needs: test + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@main + with: + template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" + build-platforms: "aws" + environment: "staging" + lfs: false + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml new file mode 100644 index 0000000..700fe7b --- /dev/null +++ b/.github/workflows/test-lint.yml @@ -0,0 +1,49 @@ +name: Test and Lint + +on: + pull_request: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + run: pip install poetry + + - name: Install dependencies + run: poetry install + + - name: Lint with flake8 + run: | + poetry run flake8 ./evaluation_function --count --select=E9,F63,F7,F82 --show-source --statistics + poetry run flake8 ./evaluation_function --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run tests + if: always() + run: poetry run pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.python-version }} + path: ./reports/pytest.xml + if-no-files-found: warn \ No newline at end of file diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml deleted file mode 100644 index 1d885ed..0000000 --- a/.github/workflows/test-report.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Test Report - -on: - workflow_run: - workflows: ["Build, Test and Deploy"] - types: - - completed - -permissions: - contents: read - actions: read - checks: write - -jobs: - report: - runs-on: ubuntu-latest - steps: - - name: Test Report - uses: dorny/test-reporter@v1 - if: always() - with: - name: Pytest Report - artifact: test-results - path: '*.xml' - reporter: java-junit - fail-on-error: false \ No newline at end of file diff --git a/README.md b/README.md index bbb9693..d9660e1 100755 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ This repository contains the boilerplate code needed to create a containerized evaluation function written in Python. +## Deployment +[![Create Release Request](https://img.shields.io/badge/Create%20Release%20Request-blue?style=for-the-badge)](https://github.com/lambda-feedback/{REPO_NAME_HERE}/issues/new?template=release-request.yml) +To deploy to production, update the README button above to point to the correct repository. + ## Quickstart This chapter helps you to quickly set up a new Python evaluation function using this template repository. @@ -98,10 +102,6 @@ shimmy -c "python" -a "-m" -a "evaluation_function.main" -i ipc ### Repository Structure ```bash -.github/workflows/ - build.yml # builds the public evaluation function image - deploy.yml # deploys the evaluation function to Lambda Feedback - evaluation_function/main.py # evaluation function entrypoint evaluation_function/evaluation.py # evaluation function implementation evaluation_function/evaluation_test.py # evaluation function tests diff --git a/evaluation_function/preview.py b/evaluation_function/preview.py index a47bcac..007f997 100755 --- a/evaluation_function/preview.py +++ b/evaluation_function/preview.py @@ -24,7 +24,5 @@ def preview_function(response: Any, params: Params) -> Result: try: return Result(preview=Preview(sympy=response)) - except FeedbackException as e: - return Result(preview=Preview(feedback=str(e))) except Exception as e: return Result(preview=Preview(feedback=str(e)))