diff --git a/.circleci/config.yml b/.circleci/config.yml
index 03aecf5f302c926c4fc42e5179bcdd3a862f8fe1..c2c65071ad7cf8bcf084a9851e0a6e9d0d75717a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -24,6 +24,21 @@ commands:
           paths:
             - vendor/bundle
 
+      # Restore cached appraisal dependencies
+      - restore_cache:
+          name: Restore bundle cache
+          key: administrate-appraisal-{{ checksum "Gemfile.lock" }}
+
+      # Install appraisal dependencies
+      - run: bundle exec appraisal install --path vendor/appraisal
+
+      # Cache appraisal dependencies
+      - save_cache:
+          name: Store bundle cache
+          key: administrate-appraisal-{{ checksum "Gemfile.lock" }}
+          paths:
+            - vendor/appraisal
+
       # Wait for DB
       - run: dockerize -wait tcp://localhost:5432 -timeout 1m
 
@@ -41,7 +56,6 @@ default_job: &default_job
   steps:
     - shared_steps
     # Run the tests against multiple versions of Rails
-    - run: bundle exec appraisal install
     - run: bundle exec appraisal rake
 
 jobs:
diff --git a/gemfiles/sass_3_4.gemfile b/gemfiles/sass_3_4.gemfile
deleted file mode 100644
index 0c0e1500b16dd287aee2620fb2397d3843372753..0000000000000000000000000000000000000000
--- a/gemfiles/sass_3_4.gemfile
+++ /dev/null
@@ -1,47 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "administrate-field-image"
-gem "autoprefixer-rails"
-gem "faker"
-gem "globalid"
-gem "pg"
-gem "redcarpet"
-gem "sentry-raven"
-gem "unicorn"
-gem "sass", "~> 3.4"
-
-group :development, :test do
-  gem "appraisal"
-  gem "awesome_print"
-  gem "bundler-audit", require: false
-  gem "byebug"
-  gem "dotenv-rails"
-  gem "factory_bot_rails"
-  gem "i18n-tasks", "0.9.29"
-  gem "pry-rails"
-  gem "rspec-rails"
-end
-
-group :test do
-  gem "ammeter"
-  gem "capybara", "3.17.0"
-  gem "database_cleaner"
-  gem "formulaic"
-  gem "launchy"
-  gem "poltergeist"
-  gem "pundit"
-  gem "shoulda-matchers"
-  gem "timecop"
-  gem "webmock"
-  gem "xpath", "3.2.0"
-end
-
-group :staging, :production do
-  gem "rack-timeout"
-  gem "rails_stdout_logging"
-  gem "uglifier"
-end
-
-gemspec path: "../"