From 720a1156cacab91cae2058ee30ca3343d54deb15 Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Tue, 28 May 2013 16:01:46 +0200 Subject: [PATCH 01/13] Added paginator which does automatic pagination and implemented it in the reports controller --- lib/adcloud.rb | 1 + lib/adcloud/connection.rb | 11 +++++++++-- lib/adcloud/paginator.rb | 35 +++++++++++++++++++++++++++++++++++ lib/adcloud/report.rb | 28 +++++++++------------------- 4 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 lib/adcloud/paginator.rb diff --git a/lib/adcloud.rb b/lib/adcloud.rb index 99f4fae..7814e95 100644 --- a/lib/adcloud.rb +++ b/lib/adcloud.rb @@ -22,6 +22,7 @@ module Adcloud autoload :Entity, "adcloud/entity" autoload :ExceptionRaiser, "adcloud/exception_raiser" autoload :MediaFile, "adcloud/media_file" + autoload :Paginator, "adcloud/paginator" autoload :Product, "adcloud/product" autoload :Report, "adcloud/report" autoload :ReportEntry, "adcloud/report_entry" diff --git a/lib/adcloud/connection.rb b/lib/adcloud/connection.rb index afb07ff..9c24c78 100644 --- a/lib/adcloud/connection.rb +++ b/lib/adcloud/connection.rb @@ -21,33 +21,40 @@ def authentication_token end def connection(auth = true) + Adcloud::logger.info "Connection Url: #{url}" + auth_header = auth && { :Authorization => "Bearer #{authentication_token}" } || {} connection ||= Faraday.new(:url => url, :headers => {}.merge(auth_header)) do |faraday| faraday.request :url_encoded # form-encode POST params - # log requests to STDOUT - faraday.response :logger if Adcloud.config.debug faraday.use ResponseErrorHandler faraday.adapter Faraday.default_adapter # make requests with Net::HTTP + # log requests to STDOUT + faraday.response :logger# if Adcloud.config.debug faraday.response :json, :content_type => /\bjson$/ end end def post(path, params = {}) + Adcloud::logger.info "POST Url: #{url} #{path}" + Adcloud::logger.info "POST params: #{params}" response = connection.post path, params response.body end def get(path, params = {}) + Adcloud::logger.info "GET Url: #{url} #{path}" response = connection.get path, params response.body end def put(path, params = {}) + Adcloud::logger.info "PUT Url: #{url} #{path}" response = connection.put path, params response.body end def delete(path, params = {}) + Adcloud::logger.info "DELETE Url: #{url} #{path}" response = connection.delete path, params response.body end diff --git a/lib/adcloud/paginator.rb b/lib/adcloud/paginator.rb new file mode 100644 index 0000000..2507e02 --- /dev/null +++ b/lib/adcloud/paginator.rb @@ -0,0 +1,35 @@ +module Adcloud + class Paginator + + def initialize(endpoint_url, opts = {}) + @endpoint_url = endpoint_url + @params = opts[:params] ||= {} + @connection = opts[:connection] ||= Connection.new + end + + def perform + paged_items = [] + + page = @params[:page] ||= 0 + total_pages = @params[:total_pages] ||= 1 + retry_count = @params[:retry_count] ||= 0 + page_result = @params[:page_result] ||= nil + + begin + begin + page += 1 + raw_result = @connection.get(@api_endpoint, @params) + total_pages = raw_result['_meta']['total_pages'] + page_result = self.new(raw_result) + paged_items += page_result.items + rescue => ex + retry if retry_count < 5 + end + end while page < total_pages + page_result.items = paged_items + page_result + end + end + + +end \ No newline at end of file diff --git a/lib/adcloud/report.rb b/lib/adcloud/report.rb index 3c18295..eb99eeb 100644 --- a/lib/adcloud/report.rb +++ b/lib/adcloud/report.rb @@ -13,25 +13,15 @@ def find_by_date(date) return self.new(result) end - def find_all_by_date(date) - paged_items = [] - page = 0 - total_pages = 1 - retry_count = 0 - page_result = nil - begin - begin - page += 1 - raw_result = connection.get(self.api_endpoint, { filter: { date: date.to_s }, :page => page, :per_page => Entity::MAX_PER_PAGE, new_backend: true }) - total_pages = raw_result['_meta']['total_pages'] - page_result = self.new(raw_result) - paged_items += page_result.items - rescue => ex - retry if retry_count < 5 - end - end while page < total_pages - page_result.items = paged_items - page_result + def find_all_by_date(date, booking_ids=[]) + params = { filter: { date: date.to_s }, :per_page => Entity::MAX_PER_PAGE, new_backend: true } + + if booking_ids + params[:new_backend] = false + end + + paginator = Adcloud::Paginator.new(self.api_endpoint, params: params ) + paginator.perform end def api_endpoint From 661e49e01fe6fa4a6b196f8edc9cef24bba8a697 Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Wed, 29 May 2013 10:46:09 +0200 Subject: [PATCH 02/13] renamed to endless_pages and made includable as mixing.. also supports passing in a block to allow for per row iteration over the result set --- lib/adcloud.rb | 2 +- lib/adcloud/endless_pages.rb | 46 ++++++++++++++++++++++++++++++++++++ lib/adcloud/paginator.rb | 35 --------------------------- 3 files changed, 47 insertions(+), 36 deletions(-) create mode 100644 lib/adcloud/endless_pages.rb delete mode 100644 lib/adcloud/paginator.rb diff --git a/lib/adcloud.rb b/lib/adcloud.rb index 7814e95..743bfc4 100644 --- a/lib/adcloud.rb +++ b/lib/adcloud.rb @@ -22,7 +22,7 @@ module Adcloud autoload :Entity, "adcloud/entity" autoload :ExceptionRaiser, "adcloud/exception_raiser" autoload :MediaFile, "adcloud/media_file" - autoload :Paginator, "adcloud/paginator" + autoload :EndlessPages, "adcloud/endless_pages" autoload :Product, "adcloud/product" autoload :Report, "adcloud/report" autoload :ReportEntry, "adcloud/report_entry" diff --git a/lib/adcloud/endless_pages.rb b/lib/adcloud/endless_pages.rb new file mode 100644 index 0000000..de71ab4 --- /dev/null +++ b/lib/adcloud/endless_pages.rb @@ -0,0 +1,46 @@ +module EndlessPages + extend ActiveSupport::Concern + + module ClassMethods + + def endless_pages(opts = {}, &block) + @endpoint_url = opts[:endpoint_url] ||= self.endpoint_url + if not @endpoint_url + raise "You must define an endpoint_url attr or pass endpoint_url into .pages(endpoint_url: 'http://www.example.com')" + end + @params = opts[:params] ||= {} + @connection = opts[:connection] ||= Adcloud::Connection.new + perform(&block) + end + + def perform(&block) + paged_items = [] + + page = @params[:page] ||= 0 + total_pages = @params[:total_pages] ||= 1 + retry_count = @params[:retry_count] ||= 0 + page_result = @params[:page_result] ||= nil + + begin + begin + page += 1 + raw_result = @connection.get(@api_endpoint, @params) + total_pages = raw_result['_meta']['total_pages'] + + page_result = self.new(raw_result) # your service class needs to include Virtus + # if a block is provided then start returning results right away + page_result.items.each do |item| + block.call(item) + end if block + paged_items += page_result.items + + rescue => ex + retry if retry_count < 5 + end + end while page < total_pages + page_result.items = paged_items + page_result + end + + end +end diff --git a/lib/adcloud/paginator.rb b/lib/adcloud/paginator.rb deleted file mode 100644 index 2507e02..0000000 --- a/lib/adcloud/paginator.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Adcloud - class Paginator - - def initialize(endpoint_url, opts = {}) - @endpoint_url = endpoint_url - @params = opts[:params] ||= {} - @connection = opts[:connection] ||= Connection.new - end - - def perform - paged_items = [] - - page = @params[:page] ||= 0 - total_pages = @params[:total_pages] ||= 1 - retry_count = @params[:retry_count] ||= 0 - page_result = @params[:page_result] ||= nil - - begin - begin - page += 1 - raw_result = @connection.get(@api_endpoint, @params) - total_pages = raw_result['_meta']['total_pages'] - page_result = self.new(raw_result) - paged_items += page_result.items - rescue => ex - retry if retry_count < 5 - end - end while page < total_pages - page_result.items = paged_items - page_result - end - end - - -end \ No newline at end of file From 02e05e824a510cf0b12f61439f8c34d62636a1d6 Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Wed, 29 May 2013 12:16:43 +0200 Subject: [PATCH 03/13] Upgraded endless_pages mixing to be cleaner --- lib/adcloud/endless_pages.rb | 44 +++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/lib/adcloud/endless_pages.rb b/lib/adcloud/endless_pages.rb index de71ab4..72e7c2c 100644 --- a/lib/adcloud/endless_pages.rb +++ b/lib/adcloud/endless_pages.rb @@ -4,27 +4,37 @@ module EndlessPages module ClassMethods def endless_pages(opts = {}, &block) - @endpoint_url = opts[:endpoint_url] ||= self.endpoint_url - if not @endpoint_url - raise "You must define an endpoint_url attr or pass endpoint_url into .pages(endpoint_url: 'http://www.example.com')" - end - @params = opts[:params] ||= {} - @connection = opts[:connection] ||= Adcloud::Connection.new + @opts = opts perform(&block) end + def conn + @opts[:connection] ||= Adcloud::Connection.new + end + + def endpoint + @opts[:endpoint_url] ||= self.endpoint_url + raise "You must either pass endpoint_url into endless_pages OR define a class method .endpoint_url" unless @opts[:endpoint_url] + @opts[:endpoint_url] + end + + def params + @opts[:params] ||= {} + end + def perform(&block) paged_items = [] - page = @params[:page] ||= 0 - total_pages = @params[:total_pages] ||= 1 - retry_count = @params[:retry_count] ||= 0 - page_result = @params[:page_result] ||= nil + page = params[:page] ||= 1 + page = 1 if page == 0 + + total_pages = 1 + retry_count = 0 + page_result = params[:page_result] ||= nil begin begin - page += 1 - raw_result = @connection.get(@api_endpoint, @params) + raw_result = conn.get(endpoint, params) total_pages = raw_result['_meta']['total_pages'] page_result = self.new(raw_result) # your service class needs to include Virtus @@ -32,12 +42,20 @@ def perform(&block) page_result.items.each do |item| block.call(item) end if block + paged_items += page_result.items + page += 1 rescue => ex - retry if retry_count < 5 + # Catch exceptions + if retry_count <= 5 + retry + else + raise ex + end end end while page < total_pages + page_result.items = paged_items page_result end From 6d1f96dd5c6ae9a7aa8af5aba8cf8c5155f6db3e Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Wed, 29 May 2013 12:17:55 +0200 Subject: [PATCH 04/13] =?UTF-8?q?Added=20booking=5Fid=20so=20that=20the=20?= =?UTF-8?q?class=20does=20not=20die=20when=20v2=20provides=20booking=5Fid?= =?UTF-8?q?=E2=80=A6=20also=20added=20accessor=20that=20will=20return=20ca?= =?UTF-8?q?mpaign=5Fid=20if=20present=20but=20if=20not=20then=20will=20use?= =?UTF-8?q?=20booking=5Fid=20so=20that=20we=20are=20forwards=20and=20backw?= =?UTF-8?q?ards=20compatible?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/adcloud/report_entry.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/adcloud/report_entry.rb b/lib/adcloud/report_entry.rb index c031a6e..e50c6e7 100644 --- a/lib/adcloud/report_entry.rb +++ b/lib/adcloud/report_entry.rb @@ -9,6 +9,7 @@ class ReportEntry attribute :product_id, Integer attribute :ad_id, Integer attribute :campaign_id, Integer + attribute :booking_id, Integer attribute :page_type, Integer attribute :ad_position, Integer attribute :ad_count, Integer @@ -44,5 +45,16 @@ class ReportEntry attribute :language, String attribute :country, String attribute :currency, String + + def campaign_id + """ + in the case of using v2 reporting system which is being depreciated + it is necessary to convert booking_id to the new campaign_id value + this method is here to ensure all services get the correct value + """ + @campaign_id || @booking_id + end end + + end \ No newline at end of file From 5465cb05c4742b69b011fc108131c415598bcad5 Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Wed, 29 May 2013 12:18:18 +0200 Subject: [PATCH 05/13] added EndlessPages Mixin and instantiated --- lib/adcloud/report.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/adcloud/report.rb b/lib/adcloud/report.rb index eb99eeb..e4fc73c 100644 --- a/lib/adcloud/report.rb +++ b/lib/adcloud/report.rb @@ -1,6 +1,7 @@ module Adcloud class Report include Virtus + include EndlessPages attribute :_meta, Hash attribute :items, Array[Adcloud::ReportEntry] @@ -20,17 +21,13 @@ def find_all_by_date(date, booking_ids=[]) params[:new_backend] = false end - paginator = Adcloud::Paginator.new(self.api_endpoint, params: params ) - paginator.perform + endless_pages(endpoint_url: self.api_endpoint, params: params) end def api_endpoint @api_endpoint ||= self.name.demodulize.tableize end - def connection - @connection ||= Connection.new - end end # Define this after class methods are added From affa35296a5a8ddf757d9cd116140a5311da403c Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Mon, 3 Jun 2013 16:25:01 +0200 Subject: [PATCH 06/13] made find_by_name valuable statically --- lib/adcloud/campaign.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adcloud/campaign.rb b/lib/adcloud/campaign.rb index 251375e..5b01d44 100644 --- a/lib/adcloud/campaign.rb +++ b/lib/adcloud/campaign.rb @@ -84,7 +84,7 @@ def valid? end # @return [Object] The entity with the unique identifier - def find_by_name(name) + def self.find_by_name(name) result = connection.get("campaigns/find_by_name", name: name) result["items"].map { |raw_campaign| self.new(raw_campaign) } end From aeaa73e38a6bf31eac95ecd495fe60fa0f5f3b4b Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Mon, 3 Jun 2013 16:25:16 +0200 Subject: [PATCH 07/13] changed logger to be debug and not info --- lib/adcloud/connection.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/adcloud/connection.rb b/lib/adcloud/connection.rb index 9c24c78..f99d56f 100644 --- a/lib/adcloud/connection.rb +++ b/lib/adcloud/connection.rb @@ -21,7 +21,7 @@ def authentication_token end def connection(auth = true) - Adcloud::logger.info "Connection Url: #{url}" + Adcloud::logger.debug "Connection Url: #{url}" auth_header = auth && { :Authorization => "Bearer #{authentication_token}" } || {} connection ||= Faraday.new(:url => url, :headers => {}.merge(auth_header)) do |faraday| @@ -35,26 +35,26 @@ def connection(auth = true) end def post(path, params = {}) - Adcloud::logger.info "POST Url: #{url} #{path}" - Adcloud::logger.info "POST params: #{params}" + Adcloud::logger.debug "POST Url: #{url} #{path}" + Adcloud::logger.debug "POST params: #{params}" response = connection.post path, params response.body end def get(path, params = {}) - Adcloud::logger.info "GET Url: #{url} #{path}" + Adcloud::logger.debug "GET Url: #{url} #{path}" response = connection.get path, params response.body end def put(path, params = {}) - Adcloud::logger.info "PUT Url: #{url} #{path}" + Adcloud::logger.debug "PUT Url: #{url} #{path}" response = connection.put path, params response.body end def delete(path, params = {}) - Adcloud::logger.info "DELETE Url: #{url} #{path}" + Adcloud::logger.debug "DELETE Url: #{url} #{path}" response = connection.delete path, params response.body end From b61e412fa7853d32affde21618284735fa13312d Mon Sep 17 00:00:00 2001 From: Ross Crawford-d'Heureuse Date: Fri, 7 Jun 2013 15:43:43 +0200 Subject: [PATCH 08/13] 1. Added to_s to adcloudapi exception so that we can just output ex and not ex.message (seems to be what everyone is doing and will help with debugging) 2. Added better exception handling.. in the else case the api was simply returning a standard error with the response.status.. which is pretty useless in some cases (ssl,timeouts,etc) so now it throws ab UnknownError that acts in the standard adcloud manner --- lib/adcloud.rb | 1 + lib/adcloud/api_error.rb | 4 ++++ lib/adcloud/exception_raiser.rb | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/adcloud.rb b/lib/adcloud.rb index 743bfc4..dd809ca 100644 --- a/lib/adcloud.rb +++ b/lib/adcloud.rb @@ -33,6 +33,7 @@ module Adcloud autoload :WebhookEvent, "adcloud/webhook_event" module AdcloudUnknownAPIError; class InvalidApiResponse < StandardError; end; end + class UnknownError < ApiError; end class BadRequestError < ApiError; end class NotFoundError < ApiError; end class ServerError < ApiError; end diff --git a/lib/adcloud/api_error.rb b/lib/adcloud/api_error.rb index 0af2d0c..bce792d 100644 --- a/lib/adcloud/api_error.rb +++ b/lib/adcloud/api_error.rb @@ -28,6 +28,10 @@ def status self.meta["status"] end + def to_s + self.response.message + end + end end \ No newline at end of file diff --git a/lib/adcloud/exception_raiser.rb b/lib/adcloud/exception_raiser.rb index 560dc24..7e97e7b 100644 --- a/lib/adcloud/exception_raiser.rb +++ b/lib/adcloud/exception_raiser.rb @@ -15,7 +15,7 @@ def initialize(response) when 500 raise Adcloud::ServerError.new(response) else - raise StandardError.new("Could not handle status #{response.status}") + raise Adcloud::UnknownError.new(response) end end From b3dff833863ca8a2f2a75a33dee449c6ba733cf5 Mon Sep 17 00:00:00 2001 From: Martin Otten Date: Mon, 10 Jun 2013 17:14:54 +0200 Subject: [PATCH 09/13] updated version --- lib/adcloud/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adcloud/version.rb b/lib/adcloud/version.rb index 279644c..7ca9455 100644 --- a/lib/adcloud/version.rb +++ b/lib/adcloud/version.rb @@ -1,3 +1,3 @@ module Adcloud - VERSION = '0.7.6.2' + VERSION = '0.7.8' end From ba7b2c147fcb587f6da6c5c492d7b02de7637433 Mon Sep 17 00:00:00 2001 From: Martin Otten Date: Wed, 12 Jun 2013 15:44:59 +0200 Subject: [PATCH 10/13] pagination skipped the last page --- .ruby-version | 1 + lib/adcloud/endless_pages.rb | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 .ruby-version diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..ae6d5b9 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +1.9.3-p392 diff --git a/lib/adcloud/endless_pages.rb b/lib/adcloud/endless_pages.rb index 3d9b9c1..6925130 100644 --- a/lib/adcloud/endless_pages.rb +++ b/lib/adcloud/endless_pages.rb @@ -33,9 +33,11 @@ def perform(&block) page_result = params[:page_result] ||= [] begin + Adcloud::logger.debug "Page: #{page}" + raw_result = conn.get(endpoint, params) total_pages = raw_result['_meta']['total_pages'] - + Adcloud::logger.debug "total_pages: #{total_pages}" page_result = self.new(raw_result) # your service class needs to include Virtus # if a block is provided then start returning results right away page_result.items.each do |item| @@ -44,7 +46,6 @@ def perform(&block) paged_items += page_result.items page += 1 - rescue => ex if retry_count <= 5 retry @@ -53,8 +54,8 @@ def perform(&block) raise ApiError, "Connection failed" end retry_count += 1 - end while page < total_pages - + end while page <= total_pages + Adcloud::logger.debug "Items: #{paged_items.length}" paged_items end From f3389676a8316815895c724175beb6c4b501fe10 Mon Sep 17 00:00:00 2001 From: Martin Otten Date: Wed, 12 Jun 2013 17:06:13 +0200 Subject: [PATCH 11/13] =?UTF-8?q?actually=20use=20the=20page=20for=20the?= =?UTF-8?q?=20pagination=20=E2=80=A6=20wicked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/adcloud/endless_pages.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/adcloud/endless_pages.rb b/lib/adcloud/endless_pages.rb index 6925130..ac1f52d 100644 --- a/lib/adcloud/endless_pages.rb +++ b/lib/adcloud/endless_pages.rb @@ -35,6 +35,8 @@ def perform(&block) begin Adcloud::logger.debug "Page: #{page}" + params[:page] = page + raw_result = conn.get(endpoint, params) total_pages = raw_result['_meta']['total_pages'] Adcloud::logger.debug "total_pages: #{total_pages}" From 9d01aa0c32e370e51c2cd394214ed8dd48f91641 Mon Sep 17 00:00:00 2001 From: Martin Otten Date: Wed, 12 Jun 2013 17:16:52 +0200 Subject: [PATCH 12/13] bumped patchlevel --- lib/adcloud/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adcloud/version.rb b/lib/adcloud/version.rb index 7ca9455..46e7cce 100644 --- a/lib/adcloud/version.rb +++ b/lib/adcloud/version.rb @@ -1,3 +1,3 @@ module Adcloud - VERSION = '0.7.8' + VERSION = '0.7.9' end From 6ed6f13e483e1bcbdd402f3f84f2497a4b8a65ec Mon Sep 17 00:00:00 2001 From: Martin Otten Date: Thu, 13 Jun 2013 17:00:40 +0200 Subject: [PATCH 13/13] added tests for endless_pages --- lib/adcloud/endless_pages.rb | 105 +++++++++++++++-------------- test/adcloud/endless_pages_test.rb | 43 ++++++++++++ 2 files changed, 97 insertions(+), 51 deletions(-) create mode 100644 test/adcloud/endless_pages_test.rb diff --git a/lib/adcloud/endless_pages.rb b/lib/adcloud/endless_pages.rb index ac1f52d..3962276 100644 --- a/lib/adcloud/endless_pages.rb +++ b/lib/adcloud/endless_pages.rb @@ -1,65 +1,68 @@ -module EndlessPages - extend ActiveSupport::Concern +module Adcloud + module EndlessPages + extend ActiveSupport::Concern - module ClassMethods + module ClassMethods - def endless_pages(opts = {}, &block) - @opts = opts - perform(&block) - end + def endless_pages(opts = {}, &block) + @opts = opts + perform(&block) + end - def conn - @opts[:connection] ||= Adcloud::Connection.new - end + def conn + @opts[:connection] ||= Adcloud::Connection.new + end - def endpoint - @opts[:endpoint_url] ||= self.endpoint_url - raise "You must either pass endpoint_url into endless_pages OR define a class method .endpoint_url" unless @opts[:endpoint_url] - @opts[:endpoint_url] - end + def endpoint + @opts[:endpoint_url] ||= self.endpoint_url + raise "You must either pass endpoint_url into endless_pages OR define a class method .endpoint_url" unless @opts[:endpoint_url] + @opts[:endpoint_url] + end - def params - @opts[:params] ||= {} - end + def params + @opts[:params] ||= {} + end - def perform(&block) - paged_items = [] + def perform(&block) + paged_items = [] - page = params[:page] ||= 1 - page = 1 if page == 0 + page = params[:page] ||= 1 + page = 1 if page == 0 - total_pages = 1 - retry_count = 0 - page_result = params[:page_result] ||= [] + total_pages = 1 + retry_count = 0 + page_result = params[:page_result] ||= [] - begin - Adcloud::logger.debug "Page: #{page}" + begin + Adcloud::logger.debug "Page: #{page}" - params[:page] = page + params[:page] = page - raw_result = conn.get(endpoint, params) - total_pages = raw_result['_meta']['total_pages'] - Adcloud::logger.debug "total_pages: #{total_pages}" - page_result = self.new(raw_result) # your service class needs to include Virtus - # if a block is provided then start returning results right away - page_result.items.each do |item| - block.call(item) - end if block + raw_result = conn.get(endpoint, params) + total_pages = raw_result['_meta']['total_pages'] || 0 + Adcloud::logger.debug "total_pages: #{total_pages}" + page_result = self.new(raw_result) # your service class needs to include Virtus + # if a block is provided then start returning results right away + page_result.items.each do |item| + block.call(item) + end if block - paged_items += page_result.items - page += 1 - rescue => ex - if retry_count <= 5 - retry - else - Adcloud.logger.error { "API Exception #{ex}" } - raise ApiError, "Connection failed" - end - retry_count += 1 - end while page <= total_pages - Adcloud::logger.debug "Items: #{paged_items.length}" - paged_items - end + paged_items += page_result.items + page += 1 + rescue => ex + Adcloud.logger.debug "Retry count: #{retry_count}" + if retry_count == 5 + retry_count += 1 + retry + else + Adcloud.logger.error { "API Exception #{ex}" } + raise StandardError, "Connection failed" + end + end while page <= total_pages + Adcloud::logger.debug "Items: #{paged_items.length}" + paged_items + end + end end -end +end \ No newline at end of file diff --git a/test/adcloud/endless_pages_test.rb b/test/adcloud/endless_pages_test.rb new file mode 100644 index 0000000..ffa130c --- /dev/null +++ b/test/adcloud/endless_pages_test.rb @@ -0,0 +1,43 @@ +require 'test_helper' + +class DummyConnection + def get(endpoint, params) + {'_meta' => {'total_pages' => 3}, 'items' => [params[:page]]} + end +end + +class Foo + include Adcloud::EndlessPages + include Virtus + attribute :_meta, Hash + attribute :items, Array + + def self.endpoint_url + "" + end +end + +describe Adcloud::EndlessPages do + subject { + Foo + } + it "raises an ApiError after 5 retries" do + stub_request(:post, "https://api.adcloud.com/v2/oauth/access_token"). + with(:body => {"client_id"=>true, "client_secret"=>true, "grant_type"=>"none"}, + :headers => {'Accept'=>'*/*', 'Content-Type'=>'application/x-www-form-urlencoded', 'User-Agent'=>'Faraday v0.8.7'}). + to_return(:status => 200, :body => "", :headers => {}) + proc { + subject.endless_pages({}) + }.must_raise StandardError + end + + it "iterates through all pages" do + result = subject.endless_pages({endpoint_url: "http://www.example.com", connection: DummyConnection.new}) + result.must_equal [1,2,3] + end + + it "gathers all results and returns a single collection" do + result = subject.endless_pages({endpoint_url: "http://www.example.com", connection: DummyConnection.new}) + result.must_equal [1,2,3] + end +end \ No newline at end of file