Pull in latest finance info automatically

.circleci/config.yml

@@ -1,18 +1,69 @@
+bundle-cache-key: &bundle-cache-key
+  v1-bundle-{{ checksum "Gemfile.lock" }}
+
+npm-cache-key: &npm-cache-key
+  v1-npm-{{ checksum "package-lock.json" }}
+
+deploy-workflow-jobs: &deploy-workflow-jobs
+  - pull-finance
+  - build:
+      requires:
+        - pull-finance
+  - deploy:
+      requires:
+        - build
+      filters:
+        branches:
+          only: master
+
+
 version: 2
 jobs:
+  pull-finance:
+    docker:
+      - image: circleci/ruby:2.3-node
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - *npm-cache-key
+      - run:
+          name: npm install
+          command: npm install
+      - save_cache:
+          key: *npm-cache-key
+          paths:
+            - node_modules
+      - run:
+          name: Clone finance data repo
+          command: git clone https://github.com/caciviclab/disclosure-backend-static.git
+          working_directory: ..
+      - run:
+          name: Clean data folder
+          command: npm run gulp -- clean
+      - run:
+          name: Pull finance data
+          command: npm run pull
+      - persist_to_workspace:
+          root: .
+          paths:
+            - _data
+
   build:
     docker:
       - image: circleci/ruby:2.3-node
     steps:
       - checkout
+      - attach_workspace:
+          at: .
       - restore_cache:
           keys:
-            - v1-bundle-{{ checksum "Gemfile.lock" }}
+            - *bundle-cache-key
       - run:
           name: Bundle Install
-          command: bundle check || bundle install
+          command: bundle check || bundle install --path vendor/bundle
       - save_cache:
-          key: v1-bundle-{{ checksum "Gemfile.lock" }}
+          key: *bundle-cache-key
           paths:
             - vendor/bundle
       - run:
@@ -46,16 +97,15 @@ jobs:
 
 workflows:
   version: 2
-  build-and-deploy:
-    jobs:
-      - build:
-          filters:
-            branches:
-              ignore: gh-pages
+  commit:
+    jobs: *deploy-workflow-jobs
 
-      - deploy:
-          requires:
-            - build
+  twice-daily:
+    triggers:
+      - schedule:
+          cron: "0 0,12 * * *"
           filters:
             branches:
-              only: master
+              only:
+                - master
+    jobs: *deploy-workflow-jobs

.gitignore

@@ -1,5 +1,10 @@
-_site
-.sass-cache
+_data/candidates
+_data/committees
+_data/contributions
+_data/referendum_opposing
+_data/referendum_supporting
+.DS_Store
 .jekyll-metadata
 node_modules
-.DS_Store
+.sass-cache
+_site

README.md

@@ -34,14 +34,12 @@ Please see [CONTRIBUTING.md](CONTRIBUTING.md) for more information.
 - [Ruby 2.3](https://www.ruby-lang.org/)
 - [Bundler](https://bundler.io/) gem
 - [Make](https://www.gnu.org/software/make/)
-
-Optionally:
-
 - [Node 8.9](https://nodejs.org/)
 
 
 ## Setup
 
+    $ npm install
     $ bundle install
     $ make build
     $ make serve
@@ -77,19 +75,11 @@ Run the tests
 ## Finance disclosures
 
 The process of downloading, converting, and calculating the reported financial
-disclosures is handled by
-[disclosure-backend-static][backend-static].
-To get the finance data into this project, we use [gulp.js](https://gulpjs.com/)
-to copy the files over and do some light transformations on them to get them
-into a [Jekyll Collection](https://jekyllrb.com/docs/collections/).
-
-We commit all the finance data to this repo so that you do not need to use the
-gulp tasks to help with this project. _You only need to run the gulp tasks if you
-want the latest finance data._
-
-This process is optional and temporary. It allows us to update the finance data
-while allowing local modifications and overrides. If you have thoughts on how
-this could be improved, [let us
+disclosures is handled by [disclosure-backend-static][backend-static].  To get
+the finance data into this project, we use [gulp.js](https://gulpjs.com/) to
+copy the files over and do some light transformations on their file paths.
+
+If you have thoughts on how this could be improved, [let us
 know](https://github.com/caciviclab/odca-jekyll/issues/5)!
 
 
@@ -101,15 +91,26 @@ directory. _It is important that the `disclosure-backend-static` project is
 cloned to the correct path (`../disclosure-backend-static`) which is hardcoded
 in the gulpfile (pull requests welcome)._
 
-Next, follow the instructions in
-[disclosure-backend-static](https://github.com/caciviclab/disclosure-backend-static/blob/master/README.md)
-and run a build. Then you can run the npm task:
+Then you can run the npm task.
 
     $ npm run pull
 
-This will copy over all files as-is and will overwrite any local modifications.
-It's important to review the changes before committing them so as not to lose
-any intentional changes.
+Now rebuild your site with the finance data.
+
+    $ make serve
+
+
+## Deployment
+
+This site is deployed by a [scheduled CI
+task](https://circleci.com/gh/caciviclab/workflows/odca-jekyll) twice daily. It
+clones the [disclosure-backend-static][backend-static] repo and pulls in the
+finance data before doing a build. Merges to the master branch will also trigger
+a deploy.
+
+The site is hosted on [GitHub Pages](https://pages.github.com/). The deploy
+consists of a force push of the `_site` directory to the `gh-pages` branch done
+by [_bin/deploy.sh](_bin/deploy.sh).
 
 
 ## License

_candidates/barbara-parker.md

@@ -1,71 +1,16 @@
 ---
-id: 1
 name: Barbara Parker
 photo_url: 'https://s3-us-west-1.amazonaws.com/odca-candidate-photos/Barbara-Parker.png'
 website_url: 'http://www.cityattorneyparker.com/'
-twitter_url: '@barbarajparker1'
+twitter_url: 'barbarajparker1'
 votersedge_url: >-
   http://votersedge.org/ca/en/ballot/election/area/42/contests/contest/13239/candidate/130764?&county=Alameda%20County&election_authority_id=1
-first_name: Barbara
-last_name: Parker
-ballot_item: 9
-office_election: 9
-bio: >-
-  Barbara Parker was appointed City Attorney in 2011 after the retirement of
-  then-City Attorney John Russo. Prior to becoming City Attorney, Parker served
-  for 20 years in the Oakland City Attorney’s Office, including ten years as
-  Chief Assistant City Attorney. Parker graduated from Harvard Law in 1975 and
-  has worked as an attorney in the private sector for two major corporations and
-  two prominent law firms. She has served as the manager of a large public
-  agency and as an attorney at all levels of government—federal, state, and
-  local—including more than five years as an Assistant United States Attorney
-  for the Northern District of California, where she represented the U.S. in
-  federal court litigation. In 2005, the State Bar Board of Governors selected
-  Parker for an appointment to the State Judicial Council. 
-
-
-  Source: http://www.cityattorneyparker.com/about and campaign website.
 committee_name: Re-elect City Attorney Barbara Parker 2016
 is_accepted_expenditure_ceiling: true
 is_incumbent: true
 occupation: 'Incumbent, City Attorney'
 party_affiliation: Democrat
 filer_id: 1382679
-supporting_money:
-  contributions_received: 67909
-  total_contributions: 67909
-  total_expenditures: 35744.47
-  total_loans_received: 0
-  contributions_by_type:
-    Individual: 59850
-    Unitemized: 393
-    Other (includes Businesses): 7666
-  contributions_by_origin:
-    Out of State: 1550
-    Within Oakland: 33391
-    Within California: 32575
-  expenditures_by_type:
-    Campaign Consultants: 31823.97
-    Campaign Paraphernalia/Misc.: 820.44
-    Candidate Filing/Ballot Fees: 1477
-    Campaign Literature and Mailings: 1559.56
-  supporting_by_type: {}
-opposing_money:
-  opposing_expenditures: null
-  opposing_by_type: {}
-contributions_received: 67909
-total_contributions: 67909
-total_expenditures: 35744.47
-total_loans_received: 0
-contributions_by_type:
-  Individual: 59850
-  Unitemized: 393
-  Other (includes Businesses): 7666
-expenditures_by_type:
-  Campaign Consultants: 31823.97
-  Campaign Paraphernalia/Misc.: 820.44
-  Candidate Filing/Ballot Fees: 1477
-  Campaign Literature and Mailings: 1559.56
 ballots:
 - _ballots/oakland/2016-11-08.md
 ---

_candidates/benjamin-lang.md

@@ -1,49 +1,15 @@
 ---
-id: 19
 name: Benjamin Lang
 photo_url: 'https://s3-us-west-1.amazonaws.com/odca-candidate-photos/Ben-Lang.png'
 website_url: 'http://www.benlang.com/'
 twitter_url: null
 votersedge_url: >-
   http://votersedge.org/ca/en/ballot/election/area/42/contests/contest/13217/candidate/130696?&county=Alameda%20County&election_authority_id=1
-first_name: Benjamin
-last_name: Lang
-ballot_item: 6
-office_election: 6
-bio: >-
-  Benjamin Lang has served as a both public school teacher and administrator for
-  over two decades. Lang has been a resident of Oakland's District 3 for 42
-  years. He holds a Bachelor’s of Arts in Political Science, a Master’s of Arts
-  in Political Theory, and a Master’s of Sciences in Educational
-  Technologies.Lang has elected to not accept any contributions for his
-  campaign.
-
-
-  Source: http://www.benlang.com/index.html
-committee_name: null
 is_accepted_expenditure_ceiling: true
 is_incumbent: false
 occupation: Retired Teacher and Administrator
 party_affiliation: Democrat
 filer_id: null
-supporting_money:
-  contributions_received: null
-  total_contributions: null
-  total_expenditures: null
-  total_loans_received: null
-  contributions_by_type: {}
-  contributions_by_origin: {}
-  expenditures_by_type: {}
-  supporting_by_type: {}
-opposing_money:
-  opposing_expenditures: null
-  opposing_by_type: {}
-contributions_received: null
-total_contributions: null
-total_expenditures: null
-total_loans_received: null
-contributions_by_type: {}
-expenditures_by_type: {}
 ballots:
 - _ballots/oakland/2016-11-08.md
 ---

_candidates/bruce-quan.md

@@ -1,84 +1,16 @@
 ---
-id: 3
 name: Bruce Quan
 photo_url: 'https://s3-us-west-1.amazonaws.com/odca-candidate-photos/Bruce_Quan.png'
 website_url: 'http://www.bruceforoakland.com/'
 twitter_url: '@BruceforOakland'
 votersedge_url: >-
   http://votersedge.org/ca/en/ballot/election/area/42/contests/contest/13234/candidate/130753?&county=Alameda%20County&election_authority_id=1
-first_name: Bruce
-last_name: Quan
-ballot_item: 3
-office_election: 3
-bio: >-
-  Bruce Quan is a retired attorney, Quan worked as the Assistant General Counsel
-  for the Redevelopment Agency of the City of San Jose and served as interim
-  City Attorney for the City of Alameda. He also served as general counsel of
-  the Organization of Chinese Americans (OCA), a national civil rights group
-  based in Washington, He co-founded a law firm, Katz, Quan, and Kors. Quan is
-  also an official with Zarsion, real estate developer and investment firm,
-  partner in Brooklyn Basin development project in the Jack London Square
-  area.Quan volunteers with the Chinese Coalition as well as Oakland Community
-  Organizations (OCO) to address issues of housing displacement, youth civic
-  leadership, and criminal justice for minorities.Quan has taught as a visiting
-  Associate Professor at Peking University Law School and as an Adjunct
-  Professor at U.C. Hastings College of Law. Quan earned his undergraduate
-  degree from UC Berkeley and a law degree from the UC Berkeley Law School.
-
-
-  Source: http://www.bruceforoakland.com/meet_bruce and
-  http://www.bruceforoakland.com/news.
 committee_name: Bruce Quan for City Council 2016
 is_accepted_expenditure_ceiling: true
 is_incumbent: false
 occupation: Real Estate Attorney
 party_affiliation: null
 filer_id: 1387267
-supporting_money:
-  contributions_received: 75623.14
-  total_contributions: 75623.14
-  total_expenditures: 75910.59
-  total_loans_received: 0
-  contributions_by_type:
-    Committee: 350
-    Individual: 31551
-    Unitemized: 1706
-    Self Funding: 39366.14
-    Other (includes Businesses): 2650
-  contributions_by_origin:
-    Out of State: 2650
-    Within Oakland: 51816.14
-    Within California: 19451
-  expenditures_by_type:
-    Office Expenses: 7808.74
-    Campaign Consultants: 52134.75
-    Meetings and Appearances: 835.15
-    Campaign Paraphernalia/Misc.: 6133.94
-    Campaign Literature and Mailings: 1056.73
-    'Professional Services (Legal, Accounting)': 5575
-    'Information Technology Costs (Internet, E-mail)': 1997.97
-  supporting_by_type: {}
-opposing_money:
-  opposing_expenditures: null
-  opposing_by_type: {}
-contributions_received: 75623.14
-total_contributions: 75623.14
-total_expenditures: 75910.59
-total_loans_received: 0
-contributions_by_type:
-  Committee: 350
-  Individual: 31551
-  Unitemized: 1706
-  Self Funding: 39366.14
-  Other (includes Businesses): 2650
-expenditures_by_type:
-  Office Expenses: 7808.74
-  Campaign Consultants: 52134.75
-  Meetings and Appearances: 835.15
-  Campaign Paraphernalia/Misc.: 6133.94
-  Campaign Literature and Mailings: 1056.73
-  'Professional Services (Legal, Accounting)': 5575
-  'Information Technology Costs (Internet, E-mail)': 1997.97
 ballots:
 - _ballots/oakland/2016-11-08.md
 ---