From 7450a607293b9634359d8eeab7bc44e39dc7a9a2 Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Thu, 26 Oct 2017 12:38:49 -0400 Subject: [PATCH 1/8] Extract endpoint superclass I'm pretty torn on this commit-an abstract superclass doesn't seem like a very ruby thing to do. Maybe I've been doing too much Java? There are two main goals of this commit: 1. It should be trivial to add new endpoints. We're going to have to add a few more, and being able to customize only the specialized bits will save a ton of time and prevent bugs. 2. Shared code should exist in one location. Already, we've got a bug in our signing process, and keeping this all in a superclass means we only need to fix it in one place. I don't like how I'm handling `@aws_credentials` at the moment (requiring each subclass to initialize it); any ideas of a better way to handle it? --- lib/amazon_product_api/endpoint.rb | 104 ++++++++++++++++++ .../item_lookup_endpoint.rb | 85 +++----------- .../item_search_endpoint.rb | 89 +++------------ 3 files changed, 132 insertions(+), 146 deletions(-) create mode 100644 lib/amazon_product_api/endpoint.rb diff --git a/lib/amazon_product_api/endpoint.rb b/lib/amazon_product_api/endpoint.rb new file mode 100644 index 0000000..76bbb5d --- /dev/null +++ b/lib/amazon_product_api/endpoint.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module AmazonProductAPI + # Base representation of all Amazon Product Advertising API endpoints. + # + # http://docs.aws.amazon.com/AWSECommerceService/latest/DG/\ + # CHAP_OperationListAlphabetical.html + # + # Any general logic relating to lookup, building the query string, + # authentication signatures, etc. should live in this class. Specializations + # (including specific request parameters and response parsing) should live in + # endpoint subclasses. + class Endpoint + require 'httparty' + require 'time' + require 'uri' + require 'openssl' + require 'base64' + + # The region you are interested in + ENDPOINT = 'webservices.amazon.com' + REQUEST_URI = '/onca/xml' + + # Generates the signed URL + def url + raise InvalidQueryError, 'Missing AWS credentials' unless aws_credentials + + "http://#{ENDPOINT}#{REQUEST_URI}" + # base + "?#{canonical_query_string}" + # query + "&Signature=#{uri_escape(signature)}" # signature + end + + # Sends the HTTP request + def get(http: HTTParty) + http.get(url) + end + + # Performs the search query and returns the processed response + def response(http: HTTParty, logger: Rails.logger) + response = parse_response get(http: http) + logger.debug response + process_response(response) + end + + private + + attr_reader :aws_credentials + + # Takes the response hash and returns the processed API response + # + # This must be implemented for each individual endpoint. + def process_response(_response_hash) + raise NotImplementedError, 'Implement this method in your subclass.' + end + + # Returns a hash of request parameters unique to the endpoint + # + # This must be implemented for each individual endpoint. + def request_params + raise NotImplementedError, 'Implement this method in your subclass.' + end + + def params + params = request_params.merge( + 'Service' => 'AWSECommerceService', + 'AWSAccessKeyId' => aws_credentials.access_key, + 'AssociateTag' => aws_credentials.associate_tag + ) + + # Set current timestamp if not set + params['Timestamp'] ||= Time.now.gmtime.iso8601 + params + end + + def parse_response(response) + Hash.from_xml(response.body) + end + + # Generates the signature required by the Product Advertising API + def signature + Base64.encode64(digest_with_key(string_to_sign)).strip + end + + def string_to_sign + "GET\n#{ENDPOINT}\n#{REQUEST_URI}\n#{canonical_query_string}" + end + + def canonical_query_string + params.sort + .map { |key, value| "#{uri_escape(key)}=#{uri_escape(value)}" } + .join('&') + end + + def digest_with_key(string) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), + aws_credentials.secret_key, + string) + end + + def uri_escape(phrase) + CGI.escape(phrase.to_s) + end + end +end diff --git a/lib/amazon_product_api/item_lookup_endpoint.rb b/lib/amazon_product_api/item_lookup_endpoint.rb index 087d597..7f026f5 100644 --- a/lib/amazon_product_api/item_lookup_endpoint.rb +++ b/lib/amazon_product_api/item_lookup_endpoint.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'amazon_product_api/endpoint' require 'amazon_product_api/lookup_response' module AmazonProductAPI @@ -7,92 +8,32 @@ module AmazonProductAPI # # http://docs.aws.amazon.com/AWSECommerceService/latest/DG/ItemLookup.html # - # Any logic relating to lookup, building the query string, authentication - # signatures, etc. should live in this class. - class ItemLookupEndpoint - require 'httparty' - require 'time' - require 'uri' - require 'openssl' - require 'base64' - - # The region you are interested in - ENDPOINT = 'webservices.amazon.com' - REQUEST_URI = '/onca/xml' - - attr_accessor :asin, :aws_credentials - + # Contains all specialization logic for this endpoint including request + # parameters, parameter validation, and response parsing. + class ItemLookupEndpoint < Endpoint def initialize(asin, aws_credentials) @asin = asin @aws_credentials = aws_credentials end - # Generate the signed URL - def url - "http://#{ENDPOINT}#{REQUEST_URI}" + # base - "?#{canonical_query_string}" + # query - "&Signature=#{uri_escape(signature)}" # signature - end - - # Send the HTTP request - def get(http: HTTParty) - http.get(url) - end - - # Performs the search query and returns the resulting SearchResponse - def response(http: HTTParty, logger: Rails.logger) - response = parse_response get(http: http) - logger.debug(response) - LookupResponse.new(response).item - end - private - def parse_response(response) - Hash.from_xml(response.body) - end + attr_reader :asin, :aws_credentials - # Generate the signature required by the Product Advertising API - def signature - Base64.encode64(digest_with_key(string_to_sign)).strip + def process_response(response_hash) + LookupResponse.new(response_hash).item end - # Generate the string to be signed - def string_to_sign - "GET\n#{ENDPOINT}\n#{REQUEST_URI}\n#{canonical_query_string}" - end - - # Generate the canonical query - def canonical_query_string - params.sort - .map { |key, value| "#{uri_escape(key)}=#{uri_escape(value)}" } - .join('&') - end - - def params - params = { - 'Service' => 'AWSECommerceService', - 'AWSAccessKeyId' => aws_credentials.access_key, - 'AssociateTag' => aws_credentials.associate_tag, - # endpoint-specific + # Other request parameters for ItemLookup can be found here: + # + # http://docs.aws.amazon.com/AWSECommerceService/latest/DG/\ + # ItemLookup.html#ItemLookup-rp + def request_params + { 'Operation' => 'ItemLookup', 'ResponseGroup' => 'ItemAttributes,Offers,Images', 'ItemId' => asin.to_s } - - # Set current timestamp if not set - params['Timestamp'] ||= Time.now.gmtime.iso8601 - params - end - - def digest_with_key(string) - OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), - aws_credentials.secret_key, - string) - end - - def uri_escape(phrase) - CGI.escape(phrase.to_s) end end end diff --git a/lib/amazon_product_api/item_search_endpoint.rb b/lib/amazon_product_api/item_search_endpoint.rb index 7230cfa..271d287 100644 --- a/lib/amazon_product_api/item_search_endpoint.rb +++ b/lib/amazon_product_api/item_search_endpoint.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'amazon_product_api/endpoint' require 'amazon_product_api/search_response' module AmazonProductAPI @@ -7,97 +8,37 @@ module AmazonProductAPI # # http://docs.aws.amazon.com/AWSECommerceService/latest/DG/ItemSearch.html # - # Any logic relating to searching, building the query string, authentication - # signatures, etc. should live in this class. - class ItemSearchEndpoint - require 'httparty' - require 'time' - require 'uri' - require 'openssl' - require 'base64' - - # The region you are interested in - ENDPOINT = 'webservices.amazon.com' - REQUEST_URI = '/onca/xml' - - attr_accessor :query, :page, :aws_credentials - + # Contains all specialization logic for this endpoint including request + # parameters, parameter validation, and response parsing. + class ItemSearchEndpoint < Endpoint def initialize(query, page, aws_credentials) + raise InvalidQueryError, "Page can't be nil." if page.nil? + @query = query @page = page @aws_credentials = aws_credentials end - # Generate the signed URL - def url - raise InvalidQueryError unless query && page - - "http://#{ENDPOINT}#{REQUEST_URI}" + # base - "?#{canonical_query_string}" + # query - "&Signature=#{uri_escape(signature)}" # signature - end - - # Send the HTTP request - def get(http: HTTParty) - http.get(url) - end - - # Performs the search query and returns the resulting SearchResponse - def response(http: HTTParty, logger: Rails.logger) - response = parse_response get(http: http) - logger.debug response - SearchResponse.new response - end - private - def parse_response(response) - Hash.from_xml(response.body) - end - - # Generate the signature required by the Product Advertising API - def signature - Base64.encode64(digest_with_key(string_to_sign)).strip - end - - # Generate the string to be signed - def string_to_sign - "GET\n#{ENDPOINT}\n#{REQUEST_URI}\n#{canonical_query_string}" - end + attr_accessor :query, :page, :aws_credentials - # Generate the canonical query - def canonical_query_string - params.sort - .map { |key, value| "#{uri_escape(key)}=#{uri_escape(value)}" } - .join('&') + def process_response(response_hash) + SearchResponse.new response_hash end - def params - params = { - 'Service' => 'AWSECommerceService', - 'AWSAccessKeyId' => aws_credentials.access_key, - 'AssociateTag' => aws_credentials.associate_tag, - # endpoint-specific + # Other request parameters for ItemLookup can be found here: + # + # http://docs.aws.amazon.com/AWSECommerceService/latest/DG/\ + # ItemSearch.html#ItemSearch-rp + def request_params + { 'Operation' => 'ItemSearch', 'ResponseGroup' => 'ItemAttributes,Offers,Images', 'SearchIndex' => 'All', 'Keywords' => query.to_s, 'ItemPage' => page.to_s } - - # Set current timestamp if not set - params['Timestamp'] ||= Time.now.gmtime.iso8601 - params - end - - def digest_with_key(string) - OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), - aws_credentials.secret_key, - string) - end - - def uri_escape(phrase) - CGI.escape(phrase.to_s) end end end From 4cd254a1cbe8ac24e6f296611c19695ab52907c8 Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Thu, 26 Oct 2017 13:05:53 -0400 Subject: [PATCH 2/8] Add Amazon Product API README Since the instructions are getting more complicated, this commit adds a README (with new endpoint instructions) to the Amazon Product API. This has three benefits: * It keeps the root README clean and relevant to most people. * It allows us to go into more documentation detail. * It facilitates an eventual extraction of the Amazon Product API lib. It might be worth extracting more from the "Getting Amazon Product Advertising API Working Locally" section into the lib README, but for now there's just endpoint information. --- README.md | 5 ++- lib/amazon_product_api/README.md | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 lib/amazon_product_api/README.md diff --git a/README.md b/README.md index fcbe52b..e82847c 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,10 @@ account or login locally: By the end of this section, you should be able to search Amazon products and add items to wishlists on your local machine (when your logged-in user is an -admin or site manager). +admin or site manager). *Note: if you're adding a new API endpoint, read more +[here][API client README].* + +[API client README]: lib/amazon_product_api/README.md **This step is only required for site managers and admins searching/adding Amazon products.** If your issue doesn't involve working with the Amazon diff --git a/lib/amazon_product_api/README.md b/lib/amazon_product_api/README.md new file mode 100644 index 0000000..1fa9cc8 --- /dev/null +++ b/lib/amazon_product_api/README.md @@ -0,0 +1,68 @@ +# Amazon Product Advertising API Client + +This folder contains the wrapper code for the Amazon Product Advertising API. +For more details on the API, check the [Amazon documentation]. + +[Amazon documentation]: http://docs.aws.amazon.com/AWSECommerceService/latest/DG/Welcome.html + +## Adding an Endpoint + +All endpoints should be subclassed from `AmazonProductAPI::Endpoint`. In order +to add a new endpoint, you'll need to modify the template below in a few ways: + + * Prepend any new attributes to the `#initialize` parameter list. Any + validations or processing can be done in `#initialize` as shown. Note: + setting `aws_credentials` is **required**! + + * Add endpoint-specific request parameters to `#request_params`. These can + be found in the Amazon API documentation. + + * Add any post-processing of the API response in `#process_response`. + + * Update the class name and comments. + +### Endpoint Template + +```ruby +require 'amazon_product_api/endpoint' + +module AmazonProductAPI + # Responsible for building and executing <...> + # + # + # + # Contains all specialization logic for this endpoint including request + # parameters, parameter validation, and response parsing. + class TemplateEndpoint < Endpoint + # Add any parameters you need for the specific endpoint. + # + # Make sure you set `@aws_credentials`-the query won't work without it! + def initialize(aws_credentials) + # Attribute validations + # raise InvalidQueryError, 'reason' if ... + + # Initialize attributes + @aws_credentials = aws_credentials + end + + private + + attr_accessor :aws_credentials # any other attrs + + # Add any post-processing of the response hash. + def process_response(response_hash) + ExampleResponse.new(response_hash).item + end + + # Include request parameters unique to this endpoint. + def request_params + { + # 'Operation' => 'ItemLookup', + # 'IdType' => 'ASIN', + # 'ItemId' => 'the item asin', + # ... + } + end + end +end +``` From 3b0bb6a629c0c16a15b4b403c224f737ba31b9e5 Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Sun, 26 Nov 2017 00:42:59 -0500 Subject: [PATCH 3/8] Only load AWS test credentials once Whenever the tests run, the following error appeared: warning: already initialized constant AWSTestCredentials This moves the AWSTestCredentials init from the individual specs to a spec helper class: `spec/support/helpers/amazon_helpers.rb`. --- spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb | 3 +-- spec/lib/amazon_product_api/item_search_endpoint_spec.rb | 3 +-- spec/support/helpers/amazon_helpers.rb | 6 ++++++ 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 spec/support/helpers/amazon_helpers.rb diff --git a/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb b/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb index c8c9288..97bcc85 100644 --- a/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb +++ b/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb @@ -1,10 +1,9 @@ # frozen_string_literal: true require 'amazon_product_api/item_lookup_endpoint' +require 'support/helpers/amazon_helpers' describe AmazonProductAPI::ItemLookupEndpoint do - AWSTestCredentials = Struct.new(:access_key, :secret_key, :associate_tag) - let(:aws_credentials) { AWSTestCredentials.new('aws_access_key', 'aws_secret_key', diff --git a/spec/lib/amazon_product_api/item_search_endpoint_spec.rb b/spec/lib/amazon_product_api/item_search_endpoint_spec.rb index 16ea7c9..190169d 100644 --- a/spec/lib/amazon_product_api/item_search_endpoint_spec.rb +++ b/spec/lib/amazon_product_api/item_search_endpoint_spec.rb @@ -1,10 +1,9 @@ # frozen_string_literal: true require 'amazon_product_api/item_search_endpoint' +require 'support/helpers/amazon_helpers' describe AmazonProductAPI::ItemSearchEndpoint do - AWSTestCredentials = Struct.new(:access_key, :secret_key, :associate_tag) - let(:aws_credentials) { AWSTestCredentials.new('aws_access_key', 'aws_secret_key', diff --git a/spec/support/helpers/amazon_helpers.rb b/spec/support/helpers/amazon_helpers.rb new file mode 100644 index 0000000..8d6b22c --- /dev/null +++ b/spec/support/helpers/amazon_helpers.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# Mocks an AWS Credentials object +AWSTestCredentials = Struct.new(:access_key, + :secret_key, + :associate_tag) From fd43cfcab91eb1786e855b8309b08ecf80717772 Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Wed, 25 Oct 2017 15:38:41 -0400 Subject: [PATCH 4/8] Add items:sync rake task `rake items:sync` will sync all items in the database. Some chunking was done in `ItemSyncJob` in order to get around Amazon's rate limitations. TODO: * Batch the Amazon query (ItemLookup can take up to 10 ASINs) * Add a delayed job queue (new issue?) * Add specs --- app/jobs/item_sync_job.rb | 61 +++++++++++++++++++++++++++++++++++++++ lib/tasks/items.rake | 8 +++++ 2 files changed, 69 insertions(+) create mode 100644 app/jobs/item_sync_job.rb create mode 100644 lib/tasks/items.rake diff --git a/app/jobs/item_sync_job.rb b/app/jobs/item_sync_job.rb new file mode 100644 index 0000000..7ec7660 --- /dev/null +++ b/app/jobs/item_sync_job.rb @@ -0,0 +1,61 @@ +require 'amazon_product_api' + +# This job is responsible for syncing items with Amazon. +# +# Amazon changes prices, details, etc. pretty frequently, so this job will +# query the Amazon record associated with the ASIN and make any required local +# updates. +# +# Bang methods (ex. `#amazon_item!`) perform an HTTP request. +# +class ItemSyncJob < ApplicationJob + queue_as :default + + # Syncs all items and writes the results to the log + def perform(*_args) + Rails.logger.info bold_green('Syncing all items') + sync_all! + Rails.logger.info bold_green('Done syncing!') + end + + private + + def sync_all! + # This is done in slices to avoid Amazon rate limits + Item.all.each_slice(3) do |items| + items.each { |item| sync! item } + sleep 2.seconds + end + end + + def sync!(item) + Rails.logger.info green("Syncing item #{item.id}: #{item.name}") + update_hash = amazon_item!(item).update_hash + item.assign_attributes(update_hash) + + if item.changed? + Rails.logger.info bold_green("Changed:\n") + item.changes.pretty_inspect + item.save! + end + end + + def amazon_item!(item) + client = AmazonProductAPI::HTTPClient.new + client.item_lookup(item.asin).response + end + + # Some styles for log text. This is so minor that it's not worth + # bringing in a full library. + + def bold_green(text) + bold(green(text)) + end + + def green(text) + "\e[32m#{text}\e[0m" + end + + def bold(text) + "\e[1m#{text}\e[22m" + end +end diff --git a/lib/tasks/items.rake b/lib/tasks/items.rake new file mode 100644 index 0000000..bf2bc8c --- /dev/null +++ b/lib/tasks/items.rake @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +namespace :items do + desc 'Check Amazon listings and update local items accordingly.' + task sync: :environment do + ItemSyncJob.perform_now + end +end From 07a6d6291317fcd27b0a45de93366cb15bbdd3a4 Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Sun, 26 Nov 2017 14:57:10 -0500 Subject: [PATCH 5/8] Rubocop fixes --- app/jobs/item_sync_job.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/jobs/item_sync_job.rb b/app/jobs/item_sync_job.rb index 7ec7660..52149c3 100644 --- a/app/jobs/item_sync_job.rb +++ b/app/jobs/item_sync_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'amazon_product_api' # This job is responsible for syncing items with Amazon. @@ -32,11 +34,10 @@ def sync!(item) Rails.logger.info green("Syncing item #{item.id}: #{item.name}") update_hash = amazon_item!(item).update_hash item.assign_attributes(update_hash) + return unless item.changed? - if item.changed? - Rails.logger.info bold_green("Changed:\n") + item.changes.pretty_inspect - item.save! - end + Rails.logger.info bold_green("Changed:\n") + item.changes.pretty_inspect + item.save! end def amazon_item!(item) From 85757bdc4946d6854871d97f8f75cd88c40faf2e Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Sun, 26 Nov 2017 14:59:37 -0500 Subject: [PATCH 6/8] Add item sync specs A couple notes: 1. I added a new exception to rubocop to allow for the `expect { action_a }.to change { dynamic_attr }` form. 2. The 2 sec Amazon rate limit delay doesn't apply to tests. All tests are green and no new rubocop violations are added. --- .rubocop.yml | 4 +++ app/jobs/item_sync_job.rb | 2 +- spec/jobs/item_sync_job_spec.rb | 54 +++++++++++++++++++++++++++++++++ spec/support/webmock.rb | 10 +++++- 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 spec/jobs/item_sync_job_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 48b1f82..4411f96 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -22,3 +22,7 @@ Style/NumericLiterals: Style/BlockDelimiters: Exclude: - spec/**/* + +Lint/AmbiguousBlockAssociation: + Exclude: + - spec/**/* diff --git a/app/jobs/item_sync_job.rb b/app/jobs/item_sync_job.rb index 52149c3..309b756 100644 --- a/app/jobs/item_sync_job.rb +++ b/app/jobs/item_sync_job.rb @@ -26,7 +26,7 @@ def sync_all! # This is done in slices to avoid Amazon rate limits Item.all.each_slice(3) do |items| items.each { |item| sync! item } - sleep 2.seconds + sleep 2.seconds unless Rails.env.test? end end diff --git a/spec/jobs/item_sync_job_spec.rb b/spec/jobs/item_sync_job_spec.rb new file mode 100644 index 0000000..aaefd2a --- /dev/null +++ b/spec/jobs/item_sync_job_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +describe ItemSyncJob do + describe '#perform' do + context 'for zero items' do + it 'runs successfully' do + expect { subject.perform }.not_to raise_error + end + end + + context 'for one item', :external do + let!(:corgi) { create(:item, asin: 'corgi_asin') } + + context 'when an item is unchanged' do + before { subject.perform } + + it 'keeps the original item' do + expect { subject.perform }.not_to change { corgi.reload.attributes } + end + end + + context 'when an item is changed' do + it 'updates the item name' do + expect { subject.perform }.to change { corgi.reload.name } + end + + it 'updates the item price' do + expect { subject.perform }.to change { corgi.reload.price_cents } + end + + it 'updates the item url' do + expect { subject.perform }.to change { corgi.reload.amazon_url } + end + + it 'updates the item image url' do + expect { subject.perform }.to change { corgi.reload.image_url } + end + end + end + + context 'for multiple items', :external do + let!(:item_1) { create(:item, asin: 'corgi_asin') } + let!(:item_2) { create(:item, asin: 'corgi_asin2') } + + it 'updates the first item' do + expect { subject.perform }.to change { item_1.reload.attributes } + end + + it 'updates the second item' do + expect { subject.perform }.to change { item_2.reload.attributes } + end + end + end +end diff --git a/spec/support/webmock.rb b/spec/support/webmock.rb index ce128c7..64bd9ea 100644 --- a/spec/support/webmock.rb +++ b/spec/support/webmock.rb @@ -7,6 +7,7 @@ RSpec.configure do |config| config.before(:each, :external) do # Item Search + search_response = file_fixture('amazon_corgi_search_response.xml').read stub_request(:get, 'webservices.amazon.com/onca/xml') .with(query: hash_including('Operation' => 'ItemSearch', @@ -14,10 +15,17 @@ .to_return(body: search_response) # Item Lookup + lookup_response = file_fixture('amazon_corgi_lookup_response.xml').read + + stub_request(:get, 'webservices.amazon.com/onca/xml') + .with(query: hash_including('Operation' => 'ItemLookup', + 'ItemId' => 'corgi_asin')) + .to_return(body: lookup_response) + stub_request(:get, 'webservices.amazon.com/onca/xml') .with(query: hash_including('Operation' => 'ItemLookup', - 'ItemId' => 'corgi_asin')) + 'ItemId' => 'corgi_asin2')) .to_return(body: lookup_response) end end From 1229548dcbad7d999909150ca45d6a15866c0ed8 Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Tue, 28 Nov 2017 19:50:59 -0500 Subject: [PATCH 7/8] Item lookup endpoint supports up to 10 ASINs As a first step for batching the requests, this commit makes the client endpoint take a list of asins instead of just an individual one. Other files are changed in order to work with the new interface (asins -> items) Next up: bulk-searching with the items sync job! --- app/jobs/item_sync_job.rb | 9 +- lib/amazon_product_api/http_client.rb | 4 +- .../item_lookup_endpoint.rb | 21 +++- lib/amazon_product_api/lookup_response.rb | 28 +++--- .../files/amazon_dual_lookup_response.xml | 5 + .../item_lookup_endpoint_spec.rb | 96 +++++++++++++------ spec/support/webmock.rb | 11 ++- 7 files changed, 122 insertions(+), 52 deletions(-) create mode 100644 spec/fixtures/files/amazon_dual_lookup_response.xml diff --git a/app/jobs/item_sync_job.rb b/app/jobs/item_sync_job.rb index 309b756..9c84a60 100644 --- a/app/jobs/item_sync_job.rb +++ b/app/jobs/item_sync_job.rb @@ -13,6 +13,10 @@ class ItemSyncJob < ApplicationJob queue_as :default + def initialize + @client = AmazonProductAPI::HTTPClient.new + end + # Syncs all items and writes the results to the log def perform(*_args) Rails.logger.info bold_green('Syncing all items') @@ -22,6 +26,8 @@ def perform(*_args) private + attr_reader :client + def sync_all! # This is done in slices to avoid Amazon rate limits Item.all.each_slice(3) do |items| @@ -41,8 +47,7 @@ def sync!(item) end def amazon_item!(item) - client = AmazonProductAPI::HTTPClient.new - client.item_lookup(item.asin).response + client.item_lookup(item.asin).response.first end # Some styles for log text. This is so minor that it's not worth diff --git a/lib/amazon_product_api/http_client.rb b/lib/amazon_product_api/http_client.rb index 427b228..c5b05f2 100644 --- a/lib/amazon_product_api/http_client.rb +++ b/lib/amazon_product_api/http_client.rb @@ -19,8 +19,8 @@ def item_search(query:, page: 1) ItemSearchEndpoint.new(query, page, aws_credentials) end - def item_lookup(asin) - ItemLookupEndpoint.new(asin, aws_credentials) + def item_lookup(*asin) + ItemLookupEndpoint.new(*asin, aws_credentials) end private diff --git a/lib/amazon_product_api/item_lookup_endpoint.rb b/lib/amazon_product_api/item_lookup_endpoint.rb index 7f026f5..ac6307a 100644 --- a/lib/amazon_product_api/item_lookup_endpoint.rb +++ b/lib/amazon_product_api/item_lookup_endpoint.rb @@ -11,17 +11,27 @@ module AmazonProductAPI # Contains all specialization logic for this endpoint including request # parameters, parameter validation, and response parsing. class ItemLookupEndpoint < Endpoint - def initialize(asin, aws_credentials) - @asin = asin + ASIN_LIMIT = 10 # max ASINs per request (from docs) + + def initialize(*asins, aws_credentials) + validate_asin_count(asins) + + @asins = asins @aws_credentials = aws_credentials end private - attr_reader :asin, :aws_credentials + attr_reader :asins, :aws_credentials + + def validate_asin_count(asins) + return unless asins.count > ASIN_LIMIT + raise ArgumentError, + "Exceeded maximum ASIN limit: #{asins.length}/#{ASIN_LIMIT}" + end def process_response(response_hash) - LookupResponse.new(response_hash).item + LookupResponse.new(response_hash).items end # Other request parameters for ItemLookup can be found here: @@ -32,7 +42,8 @@ def request_params { 'Operation' => 'ItemLookup', 'ResponseGroup' => 'ItemAttributes,Offers,Images', - 'ItemId' => asin.to_s + 'IdType' => 'ASIN', + 'ItemId' => asins.join(',') } end end diff --git a/lib/amazon_product_api/lookup_response.rb b/lib/amazon_product_api/lookup_response.rb index 160ddca..03ecb2b 100644 --- a/lib/amazon_product_api/lookup_response.rb +++ b/lib/amazon_product_api/lookup_response.rb @@ -10,34 +10,34 @@ module AmazonProductAPI # file to touch if the API response changes. class LookupResponse def initialize(response_hash) - @hash = item_hash_for response_hash + @response_hash = response_hash end - def item(item_class: SearchItem) - item_class.new(**item_attrs) + def items(item_class: SearchItem) + item_hashes.map { |hash| item_class.new(**item_attrs_from(hash)) } end private - attr_reader :hash + attr_reader :response_hash - def item_attrs + def item_attrs_from(hash) { asin: hash['ASIN'], - price_cents: hash.dig('ItemAttributes', 'ListPrice', 'Amount').to_i, + detail_page_url: hash['DetailPageURL'], + title: hash['ItemAttributes']['Title'], + price_cents: hash['ItemAttributes']['ListPrice']['Amount'].to_i, - image_url: hash.dig('SmallImage', 'URL') || '', - image_width: hash.dig('SmallImage', 'Width') || '', - image_height: hash.dig('SmallImage', 'Height') || '', - - title: hash.dig('ItemAttributes', 'Title'), - detail_page_url: hash['DetailPageURL'] + image_url: hash.dig('SmallImage', 'URL') || '', + image_width: hash.dig('SmallImage', 'Width') || '', + image_height: hash.dig('SmallImage', 'Height') || '' } end - def item_hash_for(response_hash) - response_hash.dig('ItemLookupResponse', 'Items', 'Item') || [] + def item_hashes + items = response_hash.dig('ItemLookupResponse', 'Items', 'Item') || [] + items.is_a?(Array) ? items : [items] # wrap "naked" response end end end diff --git a/spec/fixtures/files/amazon_dual_lookup_response.xml b/spec/fixtures/files/amazon_dual_lookup_response.xml new file mode 100644 index 0000000..7663bee --- /dev/null +++ b/spec/fixtures/files/amazon_dual_lookup_response.xml @@ -0,0 +1,5 @@ +
request_id0.0192698310000000TrueASINB00TFT77ZSB005R46A6WItemAttributesOffersImagesAllB00TFT77ZShttps://www.amazon.com/Douglas-1713-Toys-Louie-Corgi/dp/B00TFT77ZS?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=165953&creativeASIN=B00TFT77ZSTechnical Detailshttps://www.amazon.com/Douglas-1713-Toys-Louie-Corgi/dp/tech-data/B00TFT77ZS?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZSAdd To Baby Registryhttps://www.amazon.com/gp/registry/baby/add-item.html?asin.0=B00TFT77ZS&SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZSAdd To Wedding Registryhttps://www.amazon.com/gp/registry/wedding/add-item.html?asin.0=B00TFT77ZS&SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZSAdd To Wishlisthttps://www.amazon.com/gp/registry/wishlist/add-item.html?asin.0=B00TFT77ZS&SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZSTell A Friendhttps://www.amazon.com/gp/pdp/taf/B00TFT77ZS?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZSAll Customer Reviewshttps://www.amazon.com/review/product/B00TFT77ZS?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZSAll Offershttps://www.amazon.com/gp/offer-listing/B00TFT77ZS?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZShttps://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL._SL75_.jpg7075https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL._SL160_.jpg149160https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL.jpg209225https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL._SL30_.jpg2830https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL._SL75_.jpg7075https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL._SL75_.jpg7075https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL._SL110_.jpg102110https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL._SL160_.jpg149160https://images-na.ssl-images-amazon.com/images/I/21GKf0-rrQL.jpg209225ToyDouglas1713E1713076754813527307675481352730793631101207Douglas04729840.25911350USD$13.50Douglas11883617131713135811692557911713ToyTOYS_AND_GAMESDouglasDouglasDouglas Toys Louie Corgi767548135273767548135273793631101207Too low to display3600011https://www.amazon.com/gp/offer-listing/B00TFT77ZS?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B00TFT77ZSNewfHCFxJv3MoTibA%2FRnHuS2QSX7ZZ2Z2nf1inD%2BM4YUJSghN6aWFfEtMoIB3dQLLTKsqy1OInZM15kgcgQ%2B8gL5D5vEzn%2B2IDlxGr6NV1GLz0kmt3F88uAtHVIIxmZNCFRUCkpw7l6qwMvCdDtceXu9fR6u2yj1M4h1300USD$13.0050USD$0.504Usually ships in 24 hoursnow0011B005R46A6WB076ZTWL2Fhttps://www.amazon.com/Go-Pet-Club-Furniture-Beige/dp/B005R46A6W?psc=1&SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=165953&creativeASIN=B005R46A6WTechnical Detailshttps://www.amazon.com/Go-Pet-Club-Furniture-Beige/dp/tech-data/B005R46A6W?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6WAdd To Baby Registryhttps://www.amazon.com/gp/registry/baby/add-item.html?asin.0=B005R46A6W&SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6WAdd To Wedding Registryhttps://www.amazon.com/gp/registry/wedding/add-item.html?asin.0=B005R46A6W&SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6WAdd To Wishlisthttps://www.amazon.com/gp/registry/wishlist/add-item.html?asin.0=B005R46A6W&SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6WTell A Friendhttps://www.amazon.com/gp/pdp/taf/B005R46A6W?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6WAll Customer Reviewshttps://www.amazon.com/review/product/B005R46A6W?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6WAll Offershttps://www.amazon.com/gp/offer-listing/B005R46A6W?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6Whttps://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL._SL75_.jpg7550https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL._SL160_.jpg160107https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL.jpg500334https://images-na.ssl-images-amazon.com/images/I/41on4bk9K9L._SL30_.jpg3024https://images-na.ssl-images-amazon.com/images/I/41on4bk9K9L._SL75_.jpg7559https://images-na.ssl-images-amazon.com/images/I/41on4bk9K9L._SL75_.jpg7559https://images-na.ssl-images-amazon.com/images/I/41on4bk9K9L._SL110_.jpg11086https://images-na.ssl-images-amazon.com/images/I/41on4bk9K9L._SL160_.jpg160125https://images-na.ssl-images-amazon.com/images/I/41on4bk9K9L.jpg500392https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL._SL30_.jpg3020https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL._SL75_.jpg7550https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL._SL75_.jpg7550https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL._SL110_.jpg11073https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL._SL160_.jpg160107https://images-na.ssl-images-amazon.com/images/I/411Z%2Bw4hc%2BL.jpg500334Misc.Go Pet ClubBeige0852438003074085243800307452 High Beige Cat TreeMaterial: Compressed wood, faux fur, sisal ropeFor medium size catAssembly instruction and tools included* Overall Size : 21"W x 21"L x 52"H +* Base Board Size : 19"W x 19"L +* Size of Condo : 12"W x 12"L x 10"H +* Top of 2 Perches : 11.25" Dia +* Number of Ladder : 15200210023002100EnglishUnknownThe buyer will pay for all shipping, so it would be best that they live in San Diego so that there are no fees. We would be willing to make accommodations for the buyer if they reside in San Diego.4647USD$46.47Go Pet Club IncF56F56111422130272019611F56Pet ProductsPET_SUPPLIES19900212Go Pet Club Inc3 LevelsGo Pet Club IncGo Pet Club Cat Tree Furniture Beige8524380030748524380030743485USD$34.851100011https://www.amazon.com/gp/offer-listing/B005R46A6W?SubscriptionId=aws_access_key&tag=aws_associates_tag&linkCode=xm2&camp=2025&creative=386001&creativeASIN=B005R46A6WNewfHCFxJv3MoTibA%2FRnHuS2U3IIQqEB1yK4w3NabhLJ7QjEdeGtGBTYT7pdkXEjnT64aEnVu%2FkHEC8d27uaypZpRRlhW2Un6rQ3Wt4SLNc3r24HUS8Ha7VBw%3D%3D3485USD$34.851162USD$11.6225Usually ships in 24 hoursnow0011 diff --git a/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb b/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb index 97bcc85..d1d0b52 100644 --- a/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb +++ b/spec/lib/amazon_product_api/item_lookup_endpoint_spec.rb @@ -9,42 +9,82 @@ 'aws_secret_key', 'aws_associates_tag') } - let(:query) { - AmazonProductAPI::ItemLookupEndpoint.new('corgi_asin', aws_credentials) - } - describe '#url' do - subject(:url) { query.url } - - it { should start_with 'http://webservices.amazon.com/onca/xml' } - it { should include 'AWSAccessKeyId=aws_access_key' } - it { should include 'AssociateTag=aws_associates_tag' } - it { should include 'Operation=ItemLookup' } - it { should include 'ItemId=corgi_asin' } - it { should include 'ResponseGroup=ItemAttributes%2COffers%2CImages' } - it { should include 'Service=AWSECommerceService' } - it { should include 'Timestamp=' } - it { should include 'Signature=' } - end + context 'with one asin' do + let(:query) { + AmazonProductAPI::ItemLookupEndpoint.new('corgi_asin', aws_credentials) + } + + describe '#url' do + subject(:url) { query.url } + + it { should start_with 'http://webservices.amazon.com/onca/xml' } + it { should include 'AWSAccessKeyId=aws_access_key' } + it { should include 'AssociateTag=aws_associates_tag' } + it { should include 'Operation=ItemLookup' } + it { should include 'ItemId=corgi_asin' } + it { should include 'ResponseGroup=ItemAttributes%2COffers%2CImages' } + it { should include 'Service=AWSECommerceService' } + it { should include 'Timestamp=' } + it { should include 'Signature=' } + end - describe '#get' do - let(:http_double) { double('http') } + describe '#get' do + let(:http_double) { double('http') } - it 'should make a `get` request to the specified http library' do - expect(http_double).to receive(:get).with(String) - query.get(http: http_double) + it 'should make a `get` request to the specified http library' do + expect(http_double).to receive(:get).with(String) + query.get(http: http_double) + end + end + + describe '#response', :external do + subject(:items) { query.response } + let(:item) { items.first } + + it 'should have one entry' do + expect(items.count).to eq 1 + end + + it 'should have the item information' do + expect(item.asin).to eq 'B00TFT77ZS' + expect(item.title).to eq 'Douglas Toys Louie Corgi' + expect(item.price_cents).to eq 1350 + end end end - describe '#response', :external do - subject { query.response } + context 'with multiple asins' do + let(:query) { + AmazonProductAPI::ItemLookupEndpoint.new('corgi_asin', + 'corgi_asin2', + aws_credentials) + } + + describe '#url' do + subject(:url) { query.url } + it { should include 'ItemId=corgi_asin%2Ccorgi_asin2' } + end + + describe '#response', :external do + subject(:items) { query.response } - it { should be_a AmazonProductAPI::SearchItem } + it 'should have two entries' do + expect(items.count).to eq 2 + end + + it 'should have the item information' do + expect(items.first.asin).to eq 'B00TFT77ZS' + expect(items.second.asin).to eq 'B005R46A6W' + end + end + end - it 'should have the item information' do - expect(subject.asin).to eq 'B00TFT77ZS' - expect(subject.title).to eq 'Douglas Toys Louie Corgi' - expect(subject.price_cents).to eq 1350 + context 'for more than ten asins' do + it 'raises an argument error' do + expect { + AmazonProductAPI::ItemLookupEndpoint.new(*('1'..'11'), aws_credentials) + }.to raise_error(ArgumentError) end end end diff --git a/spec/support/webmock.rb b/spec/support/webmock.rb index 64bd9ea..5e46fc8 100644 --- a/spec/support/webmock.rb +++ b/spec/support/webmock.rb @@ -14,7 +14,7 @@ 'Keywords' => 'corgi')) .to_return(body: search_response) - # Item Lookup + # Single Item Lookup lookup_response = file_fixture('amazon_corgi_lookup_response.xml').read @@ -27,5 +27,14 @@ .with(query: hash_including('Operation' => 'ItemLookup', 'ItemId' => 'corgi_asin2')) .to_return(body: lookup_response) + + # Dual Item Lookup + + dual_lookup_response = file_fixture('amazon_dual_lookup_response.xml').read + + stub_request(:get, 'webservices.amazon.com/onca/xml') + .with(query: hash_including('Operation' => 'ItemLookup', + 'ItemId' => 'corgi_asin,corgi_asin2')) + .to_return(body: dual_lookup_response) end end From 3da732fc1fffe8a6f9015eb58dd1f45900766d12 Mon Sep 17 00:00:00 2001 From: Lee Sharma Date: Tue, 28 Nov 2017 22:08:46 -0500 Subject: [PATCH 8/8] Batch the item sync job This commit takes advantage of the multi-asin item lookup requests and performs the item sync job in batches of 10. --- app/jobs/item_sync_job.rb | 49 +++++++++++++++++++++++------- lib/amazon_product_api/endpoint.rb | 1 - 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/app/jobs/item_sync_job.rb b/app/jobs/item_sync_job.rb index 9c84a60..eb21349 100644 --- a/app/jobs/item_sync_job.rb +++ b/app/jobs/item_sync_job.rb @@ -8,9 +8,12 @@ # query the Amazon record associated with the ASIN and make any required local # updates. # -# Bang methods (ex. `#amazon_item!`) perform an HTTP request. +# Bang methods (ex. `#sync_batch!`) perform an HTTP request. # class ItemSyncJob < ApplicationJob + # Maximum number of ASINs in one request + BATCH_SIZE = AmazonProductAPI::ItemLookupEndpoint::ASIN_LIMIT + queue_as :default def initialize @@ -28,17 +31,45 @@ def perform(*_args) attr_reader :client + # Syncs all database items with their Amazon sources def sync_all! - # This is done in slices to avoid Amazon rate limits - Item.all.each_slice(3) do |items| - items.each { |item| sync! item } + # This is done in slices and batches to avoid Amazon rate limits + Item.all.each_slice(BATCH_SIZE * 3) do |batches| + batches.each_slice(BATCH_SIZE) { |batch| sync_batch! batch } sleep 2.seconds unless Rails.env.test? end end - def sync!(item) - Rails.logger.info green("Syncing item #{item.id}: #{item.name}") - update_hash = amazon_item!(item).update_hash + # Syncs one batch of items with Amazon (up to the batch size limit) + def sync_batch!(items) + count = items.count + validate_batch_size(count) + + Rails.logger.info "Fetching #{count} items: #{items.map(&:asin).join(',')}" + + items_updates = get_updates_for! items + items_updates.map { |item, updates| update_item(item, updates) } + end + + # Returns pairs of items and their corresponding update hashes + def get_updates_for!(items) + query = client.item_lookup(*items.map(&:asin)) + amazon_items = query.response + updates = amazon_items.map(&:update_hash) + + items.zip(updates) + end + + def validate_batch_size(count) + return unless count > BATCH_SIZE + raise ArgumentError, + "Batch size too large: #{count}/#{BATCH_SIZE}" + end + + def update_item(item, update_hash) + Rails.logger.info green( + "Syncing item #{item.id}: (#{item.asin}) #{item.name.truncate(64)}" + ) item.assign_attributes(update_hash) return unless item.changed? @@ -46,10 +77,6 @@ def sync!(item) item.save! end - def amazon_item!(item) - client.item_lookup(item.asin).response.first - end - # Some styles for log text. This is so minor that it's not worth # bringing in a full library. diff --git a/lib/amazon_product_api/endpoint.rb b/lib/amazon_product_api/endpoint.rb index 76bbb5d..f094066 100644 --- a/lib/amazon_product_api/endpoint.rb +++ b/lib/amazon_product_api/endpoint.rb @@ -38,7 +38,6 @@ def get(http: HTTParty) # Performs the search query and returns the processed response def response(http: HTTParty, logger: Rails.logger) response = parse_response get(http: http) - logger.debug response process_response(response) end