changes to fix document processing errors on production

Author: NickLaiacona

.env

@@ -1,3 +1,4 @@
 RAILS_ENV=development
 RACK_ENV=development
 EMAIL_DOMAIN=localhost:5000
+PORT=3000

Gemfile

@@ -18,10 +18,10 @@ gem 'high_voltage', '~> 2.1.0'
 gem "aws-sdk"
 gem "paperclip"
 gem "delayed_job_active_record"
-gem 'google_drive'
 gem 'pdf-reader'
 gem 'pdf-reader-html'
 gem 'apartment'
+gem 'yomu'
 
 group :development do
   gem 'sextant'

Gemfile.lock

@@ -66,14 +66,14 @@ GEM
     builder (3.0.4)
     callsite (0.0.11)
     cancan (1.6.10)
-    capybara (2.1.0)
+    capybara (2.5.0)
       mime-types (>= 1.16)
       nokogiri (>= 1.3.3)
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       xpath (~> 2.0)
-    capybara-webkit (1.1.1)
-      capybara (>= 2.0.2, < 2.2.0)
+    capybara-webkit (1.7.0)
+      capybara (>= 2.3.0, < 2.6.0)
       json
     carrierwave (0.10.0)
       activemodel (>= 3.2.0)
@@ -121,8 +121,6 @@ GEM
     factory_girl_rails (4.4.1)
       factory_girl (~> 4.4.0)
       railties (>= 3.0.0)
-    faraday (0.9.0)
-      multipart-post (>= 1.2, < 3)
     figaro (0.7.0)
       bundler (~> 1.0)
       rails (>= 3, < 5)
@@ -147,10 +145,6 @@ GEM
     gon (5.0.4)
       actionpack (>= 2.3.0)
       json
-    google_drive (0.3.9)
-      nokogiri (>= 1.4.4, != 1.5.2, != 1.5.1)
-      oauth (>= 0.3.6)
-      oauth2 (>= 0.5.0)
     has_scope (0.6.0.rc)
       actionpack (>= 3.2, < 5)
       activesupport (>= 3.2, < 5)
@@ -208,19 +202,11 @@ GEM
       subexec (~> 0.2.1)
     multi_json (1.10.1)
     multi_xml (0.5.5)
-    multipart-post (2.0.0)
     net-scp (1.0.4)
       net-ssh (>= 1.99.1)
     net-ssh (2.9.1)
     newrelic_rpm (3.8.1.221)
     nokogiri (1.5.11)
-    oauth (0.4.7)
-    oauth2 (0.9.4)
-      faraday (>= 0.8, < 0.10)
-      jwt (~> 1.0)
-      multi_json (~> 1.3)
-      multi_xml (~> 0.5)
-      rack (~> 1.2)
     orm_adapter (0.5.0)
     paperclip (4.1.1)
       activemodel (>= 3.0.0)
@@ -359,6 +345,9 @@ GEM
     will_paginate (3.0.5)
     xpath (2.0.0)
       nokogiri (~> 1.3)
+    yomu (0.2.2)
+      json (~> 1.8)
+      mime-types (~> 1.23)
 
 PLATFORMS
   ruby
@@ -382,7 +371,6 @@ DEPENDENCIES
   foreman
   friendly_id
   gon
-  google_drive
   high_voltage (~> 2.1.0)
   highline
   intercom-rails
@@ -418,3 +406,4 @@ DEPENDENCIES
   uglifier
   unicorn
   will_paginate (> 3.0)
+  yomu

config/database.yml

@@ -1,11 +1,11 @@
 development: &default
   adapter: postgresql
-  database: annotation_studio_development
+  database: annos
   encoding: utf8
   host: localhost
   pool: 5
   timeout: 5000
 
 test:
   <<: *default
-  database: annotation_studio_test
+  database: annos_test

config/initializers/document_processor.rb

@@ -1,5 +1 @@
-if ['development','test'].include?(Rails.env)
-  Rails.application.config.use_fake_document_processor = true
-else
   Rails.application.config.use_fake_document_processor = false
-end

lib/document_processor_dispatcher.rb

@@ -5,9 +5,9 @@ def self.processor_for(mime_type)
     end
 
     if mime_type.in?('application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')
-      GoogleDriveProcessor
+      YomuProcessor
     elsif mime_type == 'application/pdf'
-      PdfProcessor
+      YomuProcessor
     else
       NullProcessor
     end

lib/google_drive_processor.rb

@@ -1,21 +1,5 @@
 require 'rake'
-require 'google_drive_processor'
 
 namespace :annotationstudio do
-  desc 'test google drive API html conversion'
-  task drive_conversion_test: :environment do
 
-    document = Document.create(
-      title: 'Today\'s Date: ' + Time.now.to_s, author: 'Test Author',
-      upload: File.open('spec/support/example_files/annotation-studio-white-paper.docx'),
-      user_id: 2,
-      rep_group_list: "public",
-    )
-
-
-    processor = GoogleDriveProcessor.new(document, 'published')
-    processor.work
-
-    document.reload
-  end
 end

lib/tasks/annotation-studio.rake

@@ -0,0 +1,18 @@
+class YomuProcessor
+  def initialize(document, document_state)
+    @document = document
+    @original_state = document_state
+  end
+
+  def work
+    local_copy = Tempfile.new(@document.upload_file_name)  
+    @document.upload.copy_to_local_file(:original, local_copy.path)
+         
+    yomu = Yomu.new(local_copy)
+    complete = Nokogiri::HTML(yomu.html)
+    @document.text = complete.css("body").inner_html
+    @document.processed_at = DateTime.now
+    @document.state = @original_state
+    @document.save
+  end
+end

lib/yomu_processor.rb

@@ -3,27 +3,29 @@
 describe DocumentProcessorDispatcher do
   context 'when a fake is configured' do
     it 'returns the fake' do
-      expect(described_class.processor_for('blarg')).to eq ProcessorFake
+      with_real_document_processors(false) do
+        expect(described_class.processor_for('blarg')).to eq ProcessorFake
+      end
     end
   end
 
   context '.processor_for' do
-    it 'returns GoogleDriveProcessor for .doc and .docx mime-types' do
+    it 'returns YomuProcessor for .doc and .docx mime-types' do
       with_real_document_processors do
         %w(
 application/msword
 application/vnd.openxmlformats-officedocument.wordprocessingml.document
         ).each do |mime_type|
 
-          expect(described_class.processor_for(mime_type)).to eq GoogleDriveProcessor
+          expect(described_class.processor_for(mime_type)).to eq YomuProcessor
         end
       end
     end
 
-    it 'returns PdfProcessor for pdf mime-types' do
+    it 'returns YomuProcessor for pdf mime-types' do
       with_real_document_processors do
         expect(described_class.processor_for('application/pdf')).to eq \
-          PdfProcessor
+          YomuProcessor
       end
     end
 
@@ -34,12 +36,14 @@
     end
   end
 
-  def with_real_document_processors
+  def with_real_document_processors( use_real=true )
+    default_setting = Rails.application.config.use_fake_document_processor
     begin
-      Rails.application.config.use_fake_document_processor = false
+      Rails.application.config.use_fake_document_processor = (not use_real)
       yield
     ensure
-      Rails.application.config.use_fake_document_processor = true
+      Rails.application.config.use_fake_document_processor = default_setting
     end
   end
-end
+  
+end