build first set of docker container in pipline and place them in the the github registry (#142)

- added a Dockerfile for the thre containers
- added a workflow step to build and placing the container in the
registry
- added a docker-compose.yaml to use the build images
- added a docker compose to build the images locally and a script for
prebuild steps

.github/workflows/delete-branch.yaml

@@ -0,0 +1,11 @@
+name: Branch Deleted
+on: delete
+jobs:
+  delete:
+    if: github.event.ref_type == 'branch'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clean up
+      run: |
+        echo "Clean up for branch ${{ github.event.ref }}"
+       # uses: bots-house/ghcr-delete-image-action@v1.1.0

.github/workflows/documentation.yaml

@@ -5,7 +5,7 @@ on:
     branches:
     - main
   pull_request:
-    types: [opened, reopened, synchronize]
+    types: [reopened, opened, synchronize]
 
 jobs:
   doc-build:
@@ -23,7 +23,7 @@ jobs:
       with:
         version: 1.4.2
         virtualenvs-create: false
-    - run: poetry install --only doc,root,develop
+    - run: poetry install --only doc,root,develop --all-extras
     - name: Doc-Build
       run: |
         cd documentations

.github/workflows/lint-actions.yaml

@@ -37,7 +37,7 @@ jobs:
         version: 1.4.2
         virtualenvs-create: false
         virtualenvs-path: ~/local/share/virtualenvs
-    - run: poetry install --without develop,doc
+    - run: poetry install --without develop,doc --all-extras
     - name: Run mypy
       run: |
         mypy src tests
@@ -69,7 +69,6 @@ jobs:
       uses: actions/checkout@v3
     - name: Install root, pip-licenses & pip-audit
       run: |
-        poetry install --only-root
         poetry run pip install pip-licenses pip-audit
     - name: Poetry export
       run: poetry export -f requirements.txt --output requirements.txt

.github/workflows/test-and-build-action.yaml

@@ -1,9 +1,12 @@
 name: Test & Build
 
 on:
+  push:
   pull_request:
     types: [reopened, opened, synchronize]
-  push:
+  release:
+    types:
+    - created
 
 jobs:
   test:
@@ -27,7 +30,7 @@ jobs:
       with:
         path: ~/.local/share/virtualenvs
         key: venv
-    - run: poetry install --without develop,doc,lint
+    - run: poetry install --without develop,doc,lint --all-extras
     - name: Run test suite
       run: |
         poetry run pytest --junit-xml=unit-test-results.xml --cov-report "xml:coverage.xml" --cov=src tests/
@@ -85,7 +88,7 @@ jobs:
         version: 1.4.2
         virtualenvs-path: ~/local/share/virtualenvs
     - run: |
-        poetry install --only test
+        poetry install --only test --all-extras
     - uses: actions/download-artifact@v3
       with:
         name: code-coverage-report
@@ -99,9 +102,9 @@ jobs:
         path: htmlcov/
 
   build:
+    if: ${{ github.event_name == 'push' }}
     runs-on: ubuntu-latest
     needs: test
-    if: ${{ github.event_name == 'push' }}
     steps:
     - name: Setup Python
       uses: actions/setup-python@v4
@@ -120,10 +123,87 @@ jobs:
     - name: Check out repository code
       uses: actions/checkout@v3
     - run: |
-        poetry install --without develop,doc,lint,test
         poetry build
     - name: Archive builds
       uses: actions/upload-artifact@v3
       with:
-        name: builds
+        name: wheel-builds
         path: dist/
+
+  docker-build:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+    - name: Check out repository code
+      uses: actions/checkout@v3
+
+    - uses: actions/download-artifact@v3
+      with:
+        name: wheel-builds
+        path: dist
+
+    - id: tag_or_main
+      run: |
+        if [ "$GITHUB_EVENT_NAME" = "release" ]; then
+          echo "TAG_OR_MAIN=$(echo $GITHUB_REF | awk -F / '{print $NF}')" >> $GITHUB_ENV
+          echo "LATEST=latest" >> $GITHUB_ENV
+        elif [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then
+          echo "TAG_OR_MAIN=main" >> $GITHUB_ENV
+          echo "LATEST=main" >> $GITHUB_ENV
+        else
+          echo "TAG_OR_MAIN=experimental" >> $GITHUB_ENV
+          echo "LATEST=experimental" >> $GITHUB_ENV
+        fi
+
+
+    - run: |
+        echo "Print ${{ env.TAG_OR_MAIN }}"
+
+    - name: Build All prior to push
+      run: |
+        docker build --target ingest .
+        docker build --target data-transformation .
+        docker build --target web-server .
+        docker image ls
+
+    - name: Exit workflow on pull request
+      if: ${{ github.event_name == 'pull_request' }}
+      run: exit 0
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build and push ingest
+      uses: docker/build-push-action@v4
+      with:
+        push: true
+        tags: |
+          ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/ingest:${{ env.TAG_OR_MAIN }}
+          ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/ingest:${{ env.LATEST }}
+        target: ingest
+        context: .
+
+    - name: Build and push data-transformation
+      uses: docker/build-push-action@v4
+      with:
+        push: true
+        tags: |
+          ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/data-transformation:${{ env.TAG_OR_MAIN }}
+          ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/data-transformation:${{ env.LATEST }}
+        target: data-transformation
+        context: .
+
+    - name: Build and push web-server
+      uses: docker/build-push-action@v4
+      with:
+        push: true
+        tags: |
+          ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/web-server:${{ env.TAG_OR_MAIN }}
+          ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/web-server:${{ env.LATEST }}
+        target: web-server
+        context: .