diff --git a/.github/workflows/code-quality.yaml b/.github/workflows/code-quality.yaml
deleted file mode 100644
index 77f1b55e0e..0000000000
--- a/.github/workflows/code-quality.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-name: Code Quality Checks
-on:
-  push:
-    branches:
-    - main
-    - release/**
-  pull_request:
-    branches:
-    - main
-    - release/**
-  workflow_call:
-  workflow_dispatch:
-# Cancel old runs when a new commit is pushed to the same branch if not on main or dev
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
-defaults:
-  run:
-    working-directory: .
-jobs:
-  code-quality:
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
-    strategy:
-      matrix:
-        python_version:
-        - '3.9'
-        - '3.10'
-        pip_deps:
-        - '[dev]'
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python_version }}
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        python -m pip install --upgrade .${{ matrix.pip_deps }}
-    - name: Run checks
-      run: |
-        pre-commit run --all-files
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
deleted file mode 100644
index 86889e6402..0000000000
--- a/.github/workflows/codeql-analysis.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: 'CodeQL'
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [main]
-  schedule:
-  - cron: '0 9 * * 1'  # Every Monday at 09:00 (9:00 AM)
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ['python']
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        setup-python-dependencies: false
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    # - run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
deleted file mode 100644
index f89d67ec39..0000000000
--- a/.github/workflows/coverage.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: PyTest Coverage
-on:
-  workflow_call:
-    inputs:
-      download-path:
-        required: true
-        type: string
-jobs:
-  coverage:
-    timeout-minutes: 5
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Repo
-      uses: actions/checkout@v3
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        pip install coverage[toml]==6.5.0
-    - name: Download artifacts
-      uses: actions/download-artifact@v3
-      with:
-        path: ${{ inputs.download-path }}
-    - name: Generate coverage report
-      run: |
-        set -ex
-
-        # Flatten the coverage files
-        ls ${{ inputs.download-path }} | while read x; do mv ${{ inputs.download-path }}/$x/.coverage .coverage.$x; done
-
-        python -m coverage combine
-        python -m coverage report
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index bbc5e422d3..e0b2de4335 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -17,18 +17,9 @@ jobs:
     strategy:
       matrix:
         include:
-        - name: '2.1.0_cu121'
-          base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04
-          dep_groups: '[gpu]'
         - name: '2.1.0_cu121_flash2'
           base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04
           dep_groups: '[gpu-flash2]'
-        - name: '2.1.0_cu121_aws'
-          base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04-aws
-          dep_groups: '[gpu]'
-        - name: '2.1.0_cu121_flash2_aws'
-          base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04-aws
-          dep_groups: '[gpu-flash2]'
     steps:
     - name: Maximize Build Space on Worker
       uses: easimon/maximize-build-space@v4
@@ -81,12 +72,13 @@ jobs:
 
     - name: Build and Push the Docker Image
       uses: docker/build-push-action@v3
-      with:
+      with:       
         context: .
         tags: ${{ env.IMAGE_TAG }}
         push: true
         cache-from: type=registry,ref=${{ env.IMAGE_CACHE }}
         cache-to: type=registry,ref=${{ env.IMAGE_CACHE }},mode=max
         build-args: |
+          BRANCH_NAME=${{ github.head_ref || github.ref_name }} 
           BASE_IMAGE=${{ matrix.base_image }}
           DEP_GROUPS=${{ matrix.dep_groups }}
diff --git a/.github/workflows/pr-cpu.yaml b/.github/workflows/pr-cpu.yaml
deleted file mode 100644
index c5bb4d641c..0000000000
--- a/.github/workflows/pr-cpu.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: PR CPU tests
-on:
-  push:
-    branches:
-    - main
-    - release/*
-  pull_request:
-    branches:
-    - main
-    - release/*
-  workflow_dispatch:
-# Cancel old runs when a new commit is pushed to the same branch if not on main or dev
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
-jobs:
-  pytest-cpu:
-    uses: ./.github/workflows/pytest-cpu.yaml
-    strategy:
-      matrix:
-        include:
-        - name: 'cpu-2.1.0'
-          container: mosaicml/pytorch:2.1.0_cpu-python3.10-ubuntu20.04
-          markers: 'not gpu'
-          pytest_command: 'coverage run -m pytest'
-    name: ${{ matrix.name }}
-    if: github.repository_owner == 'mosaicml'
-    with:
-      container: ${{ matrix.container }}
-      name: ${{ matrix.name }}
-      pytest-command: ${{ matrix.pytest_command }}
-      pytest-markers: ${{ matrix.markers }}
-  coverage:
-    uses: ./.github/workflows/coverage.yaml
-    name: Coverage Results
-    if: github.repository_owner == 'mosaicml'
-    needs: [pytest-cpu]
-    with:
-      download-path: artifacts
diff --git a/.github/workflows/pr-gpu.yaml b/.github/workflows/pr-gpu.yaml
deleted file mode 100644
index 7d9320f9e0..0000000000
--- a/.github/workflows/pr-gpu.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-name: PR GPU tests
-on:
-  push:
-    branches:
-    - main
-    - release/*
-  pull_request_target:
-    branches:
-    - main
-    - release/**
-  workflow_dispatch:
-# Cancel old runs when a new commit is pushed to the same branch if not on main or dev
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
-jobs:
-  pytest-gpu:
-    uses: ./.github/workflows/pytest-gpu.yaml
-    strategy:
-      matrix:
-        include:
-        - name: 'gpu-2.1.0'
-          container: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04
-          markers: 'gpu'
-          pytest_command: 'coverage run -m pytest'
-          deps_group: 'all'
-        - name: 'gpu-2.1.0-flash2'
-          container: mosaicml/llm-foundry:2.1.0_cu121_flash2-latest
-          markers: 'gpu'
-          pytest_command: 'coverage run -m pytest'
-          deps_group: 'all-flash2'
-    name: ${{ matrix.name }}
-    if: github.repository_owner == 'mosaicml'
-    with:
-      container: ${{ matrix.container }}
-      mcloud-timeout: 1800
-      name: ${{ matrix.name }}
-      pytest-command: ${{ matrix.pytest_command }}
-      pytest-markers: ${{ matrix.markers }}
-      python-version: 3.9
-      deps-group: ${{ matrix.deps_group }}
-    secrets:
-      mcloud-api-key: ${{ secrets.MCLOUD_API_KEY }}
diff --git a/.github/workflows/pytest-cpu.yaml b/.github/workflows/pytest-cpu.yaml
deleted file mode 100644
index 3bb3697b91..0000000000
--- a/.github/workflows/pytest-cpu.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: Pytest CPU
-on:
-  workflow_call:
-    inputs:
-      container:
-        required: true
-        type: string
-      name:
-        required: true
-        type: string
-      pytest-command:
-        required: true
-        type: string
-      pytest-markers:
-        required: true
-        type: string
-jobs:
-  pytest-cpu:
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    container: ${{ inputs.container }}
-    steps:
-    - name: Checkout Repo
-      uses: actions/checkout@v3
-    - name: Setup
-      run: |
-        set -ex
-        export PATH=/composer-python:$PATH
-        python -m pip install --upgrade 'pip<23' wheel
-        python -m pip install --upgrade .[all-cpu]
-    - name: Run Tests
-      id: tests
-      run: |
-        set -ex
-        export PATH=/composer-python:$PATH
-        export COMMON_ARGS="-v --durations=20 -m '${{ inputs.pytest-markers }}' -o tmp_path_retention_policy=none"
-
-        # Necessary to run git diff for doctests
-        git config --global --add safe.directory /__w/llm-foundry/llm-foundry
-
-        make test PYTEST='${{ inputs.pytest-command }}' EXTRA_ARGS="$COMMON_ARGS --codeblocks"
-        # make test-dist PYTEST='${{ inputs.pytest-command }}' EXTRA_ARGS="$COMMON_ARGS" WORLD_SIZE=2
-
-        python -m coverage combine
-    - uses: actions/upload-artifact@v3
-      with:
-        name: coverage-${{ github.sha }}-${{ inputs.name }}
-        path: .coverage
diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml
deleted file mode 100644
index ec123c26a5..0000000000
--- a/.github/workflows/pytest-gpu.yaml
+++ /dev/null
@@ -1,83 +0,0 @@
-name: Pytest GPU
-on:
-  workflow_call:
-    inputs:
-      container:
-        required: true
-        type: string
-      mcloud-timeout:
-        required: false
-        type: number
-        default: 1800
-      name:
-        required: true
-        type: string
-      pytest-command:
-        required: true
-        type: string
-      pytest-markers:
-        required: true
-        type: string
-      python-version:
-        required: false
-        type: string
-        default: 3.9
-      deps-group:
-        required: true
-        type: string
-    secrets:
-      mcloud-api-key:
-        required: true
-jobs:
-  pytest-gpu:
-    timeout-minutes: 60 # ${{ inputs.gha-timeout }} for some reason not able to turn this into an input
-    runs-on: ubuntu-latest
-    env:
-      MOSAICML_API_KEY: ${{ secrets.mcloud-api-key }}
-    steps:
-    - name: Checkout Repo
-      uses: actions/checkout@v3
-    - name: Setup Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ inputs.python-version }}
-    - name: Cache pip
-      uses: actions/cache@v3
-      with:
-        # This path is specific to Ubuntu
-        path: ~/.cache/pip
-        # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-          ${{ runner.os }}-
-    - name: Setup MCLI
-      run: |
-        set -ex
-        python -m pip install mosaicml-cli
-        mcli version
-    - name: Submit Run
-      id: tests
-      run: |
-        set -ex
-
-        PR_NUMBER="$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")"
-        REF_ARGS=""
-
-        # Use the PR number if it exists, commit SHA for protected branches and the branch name otherwise
-        if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then
-          if [[ "$GITHUB_REF" =~ "refs/heads/main" || "$GITHUB_REF" =~ "refs/heads/release" ]]; then
-            REF_ARGS="--git_commit $GITHUB_SHA"
-          else
-            REF_ARGS="--git_branch $GITHUB_REF_NAME"
-          fi
-        else
-          REF_ARGS="--pr_number $PR_NUMBER"
-        fi
-
-        python .github/mcp/mcp_pytest.py \
-              --image '${{ inputs.container }}' \
-              --pytest_markers '${{ inputs.pytest-markers }}' \
-              --pytest_command '${{ inputs.pytest-command }}' \
-              --timeout ${{ inputs.mcloud-timeout }} ${REF_ARGS} \
-              --deps_group ${{ inputs.deps-group }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
deleted file mode 100644
index 4f241323b9..0000000000
--- a/.github/workflows/release.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-name: Release
-
-on:
-  push:
-    tags:
-    - 'v*'
-  workflow_dispatch:
-
-jobs:
-  code-quality:
-    uses: ./.github/workflows/code-quality.yaml
-
-  pypi-packaging:
-    name: Build and Publish llm-foundry PyPI Package
-    needs:
-    - code-quality
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout source
-      uses: actions/checkout@v3
-
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: '3.9'
-
-    - name: Build source and wheel distributions
-      run: |
-        if [[ "${{ github.ref }}" =~ refs\/tags\/v ]]; then
-          PYPI_PACKAGE_NAME="llm-foundry"
-        else
-          PYPI_PACKAGE_NAME="llm-foundry-test-$(date +%Y%m%d%H%M%S)"
-        fi
-
-        # Remove the peft, xentropy-cuda-lib and triton-pre-mlir dependencies as PyPI does not
-        # support direct installs. The error message for importing PEFT, FusedCrossEntropy,
-        # and flash_attn_triton gives instructions on how to install if a user tries to use it
-        # without this dependency.
-        sed '/xentropy-cuda-lib@git+https:\/\/github.com\/HazyResearch\/flash-attention.git@.*/d' -i setup.py
-        sed '/triton-pre-mlir@git+https:\/\/github.com\/vchiley\/triton.git@.*/d' -i setup.py
-        sed '/peft@git+https:\/\/github.com\/huggingface\/peft.git.*/d' -i setup.py
-
-        python -m pip install --upgrade build twine
-        python -m build
-        twine check --strict dist/*
-
-    - name: Publish 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      if: contains(github.ref, 'refs/tags/v')
-      with:
-        user: __token__
-        password: ${{ secrets.PROD_PYPI_API_TOKEN }}
-
-    - name: Publish distribution 📦 to Test PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      if: contains(github.ref, 'refs/heads/') || contains(github.ref, 'refs/pull/')
-      with:
-        user: __token__
-        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
-        repository_url: https://test.pypi.org/legacy/
diff --git a/.github/workflows/smoketest.yaml b/.github/workflows/smoketest.yaml
deleted file mode 100644
index 0bf3968753..0000000000
--- a/.github/workflows/smoketest.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-name: Smoketest
-on:
-  push:
-    branches:
-    - main
-    - release/*
-  pull_request:
-    branches:
-    - main
-    - release/*
-  workflow_dispatch:
-# Cancel old runs when a new commit is pushed to the same branch if not on main or dev
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
-defaults:
-  run:
-    working-directory: .
-jobs:
-  smoketest:
-    runs-on: ubuntu-20.04
-    timeout-minutes: 10
-    strategy:
-      matrix:
-        python_version:
-        - "3.9"
-        - "3.10"
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python_version }}
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        python -m pip install --upgrade .
-        python -m pip install pytest==7.2.1 pytest_codeblocks==0.16.1
-    - name: Run checks
-      run: |
-        pytest tests/test_smoketest.py
diff --git a/Dockerfile b/Dockerfile
index 6c283660c4..29984cd263 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,9 +5,14 @@ ARG BASE_IMAGE
 FROM $BASE_IMAGE
 
 ARG DEP_GROUPS
+ARG BRANCH_NAME
 
 # Install and uninstall foundry to cache foundry requirements
-RUN git clone -b main https://github.com/mosaicml/llm-foundry.git
-RUN pip install --no-cache-dir "./llm-foundry${DEP_GROUPS}"
+COPY ./setup.py ./setup.py
+RUN git clone -b $BRANCH_NAME https://github.com/mosaicml/llm-foundry.git
+RUN pip install --no-cache-dir --upgrade "./llm-foundry${DEP_GROUPS}"
 RUN pip uninstall -y llm-foundry
 RUN rm -rf llm-foundry
+RUN rm ./setup.py
+
+RUN pip freeze
diff --git a/llmfoundry/data/packing.py b/llmfoundry/data/packing.py
index d3084c72c8..70fe55e40f 100644
--- a/llmfoundry/data/packing.py
+++ b/llmfoundry/data/packing.py
@@ -524,7 +524,5 @@ def parse_args() -> Namespace:
     header = '\n\n\n packing_ratio | % PADDING | % WASTE'
     fstr = '        {:5.1f}  |  {:5.2f}%   | {:6.2f}%'
 
-    print(header)
-    print('-' * len(header))
     for packing_ratio, padding, waste in results:
         print(fstr.format(packing_ratio, padding, waste))
diff --git a/llmfoundry/utils/builders.py b/llmfoundry/utils/builders.py
index 42f817b386..3baf885468 100644
--- a/llmfoundry/utils/builders.py
+++ b/llmfoundry/utils/builders.py
@@ -392,6 +392,7 @@ def build_tokenizer(
         # Make sure the tokenizer files are downloaded and cached first by local rank 0
         with dist.local_rank_zero_download_and_wait(signal_file_path):
             pass
+    print('hello')
 
     if tokenizer_name.startswith('tiktoken'):
         tokenizer = TiktokenTokenizerWrapper(**tokenizer_kwargs)
diff --git a/setup.py b/setup.py
index e5bc7e81d2..74ca5bfd29 100644
--- a/setup.py
+++ b/setup.py
@@ -105,7 +105,7 @@
     'xentropy-cuda-lib@git+https://github.com/HazyResearch/flash-attention.git@v1.0.9#subdirectory=csrc/xentropy',
 ]
 extra_deps['gpu-flash2'] = [
-    'flash-attn==2.4.2',
+    'flash-attn==2.3.2',
     'mosaicml-turbo==0.0.8',
 ]