Pagination of Results

About

In most search application usage, the top matching results (sorted by relevance, or some other criteria) are then displayed to some human user. In many applications the UI for these sorted results are displayed to the user in pages containing a fixed number of matching results, and users don't typically look at results past the first few pages worth of results.

Fetching A Large Number of Sorted Results: Cursor

The API supports using a cursor to scan through results. Cursor in the API is a logical concept, that doesn't involve caching any state information on the server. Instead the sort values of the last story returned to the client are used to compute a next_page_cursor representing a logical point in the ordered space of sort values. That next_page_cursor can be specified in the parameters of subsequent requests to tell the API where to continue.

Using Cursor

To use a cursor with the API, specify a cursor parameter with the value of *. You can think of this being analogous to page=1 as a way to tell the API "start at the beginning of my sorted results" except that it also informs the API that you want to use a Cursor. Although, the default value of cursor is * unless you specify it. So in addition to returning the top N sorted results (where you can control N using the per_page parameter) the API response will also include an encoded String named next_page_cursor. You then take the next_page_cursor String value from the response, and pass it back to the API as the cursor parameter for your next request. You can repeat this process until you've fetched as many stories as you want, or until the next_page_cursor returned matches the cursor you've already specified — indicating that there are no more results.

Using Per Page

The API supports using a per_page to specify the maximum number of stories per page. This parameter is used to paginate results from a query.

Examples

The following example shows how to retrieve all stories in English, are about Sports and were published between 1 hour ago and now.

var AylienNewsApi = require('aylien-news-api');
var Promise = require('bluebird');

// Async Promise “while loop”
function promiseWhile(condition, action) {
  var resolver = Promise.defer();

  var loop = function() {
    if (!condition()) return resolver.resolve();
    return Promise.cast(action())
      .then(loop)
      .catch(resolver.reject);
  };

  process.nextTick(loop);

  return resolver.promise;
}

// Async Promise “sleep”
function sleep(time){
  return new Promise((resolve) => setTimeout(resolve, time));
}

function fetchNewStories(opts){
  return new Promise(function (resolve, reject){
    var fetchedStories = [];
    var stories = null;
    var whileCondition = function(){
      return (stories == null || stories.length > 0);
    };
    
    promiseWhile(whileCondition, function(){
      return new Promise(function (resolve, reject){
        apiInstance.listStories(opts, function(error, data, response){
          if (error){
            if (response.status == 429) {
              console.log('Usage limit are exceeded. Wating for 30 seconds...');
              sleep(30 * 1000).then(function(){
                resolve();
              });
            } else {
              reject(error);
            }
          } else {
            opts.cursor = data.nextPageCursor;
            stories = data.stories;
            
            fetchedStories = fetchedStories.concat(stories);
            
            console.log("Fetched " + stories.length +
              " stories. Total story count so far: " + fetchedStories.length);
            
            resolve();
          }
        });
      });
    }).then(function(){
      resolve(fetchedStories);
    });
  });
}


var apiInstance = new AylienNewsApi.DefaultApi();

// Configure API key authorization: app_id
var app_id = apiInstance.apiClient.authentications['app_id'];
app_id.apiKey = "YOUR_APP_ID";

// Configure API key authorization: app_key
var app_key = apiInstance.apiClient.authentications['app_key'];
app_key.apiKey = "YOUR_APP_KEY";

var opts = {
  'language': ['en'],
  'publishedAtStart': 'NOW-1HOUR',
  'publishedAtEnd': 'NOW', 
  'categoriesTaxonomy': 'iab-qag',
  'categoriesId': ['IAB17'],
  'cursor': '*',
  'perPage': 16
};

fetchNewStories(opts).then(function(stories){
  console.log('**************');
  console.log('Fetched ' + stories.length +
    ' stories which are in English, are about Sports and were' +
    ' published between 1 hour ago and now');
});
import time
import aylien_news_api
from aylien_news_api.rest import ApiException

def fetch_new_stories(params={}):
  fetched_stories = []
  stories = None
  
  while stories is None or len(stories) > 0:
    try:
      response = api_instance.list_stories(**params)
    except ApiException as e:
      if ( e.status == 429 ):
        print('Usage limit are exceeded. Wating for 60 seconds...')
        time.sleep(60)
        continue
        
    stories = response.stories
    params['cursor'] = response.next_page_cursor
    
    fetched_stories += stories
    print("Fetched %d stories. Total story count so far: %d" %
      (len(stories), len(fetched_stories)))
     
  return fetched_stories
  
# Configure API key authorization: app_id
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = 'YOUR_APP_ID'
# Configure API key authorization: app_key
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = 'YOUR_APP_KEY'

# create an instance of the API class
api_instance = aylien_news_api.DefaultApi()

params = {
  'language': ['en'],
  'published_at_start': 'NOW-1HOUR',
  'published_at_end': 'NOW',
  'categories_taxonomy': 'iab-qag',
  'categories_id': ['IAB17'],
  'cursor': '*',
  'per_page': 16
}

stories = fetch_new_stories(params)

print('************')
print("Fetched %d stories which are in English, are \
about Sports and were published between %s and %s" %
(len(stories), params['published_at_start'], params['published_at_end']))
<?php
require_once(__DIR__ . '/vendor/autoload.php');

function fetch_new_stories($opts){
  global $api_instance;
  
  $fetched_stories = [];
  $stories = NULL;
  
  while (is_null($stories) || count($stories) > 0) {
    try {
      $result = $api_instance->listStories($opts);
    } catch (Exception $e) {
      $code = $e->getResponseObject()->getErrors()[0]->getStatus();
      
      if ($code == '429'){
        print_r("Usage limit are exceeded. Wating for 30 seconds...\n");
        sleep(30);
        continue;
      }
    }
    
    $stories = $result->getStories();
    $fetched_stories = array_merge($fetched_stories, $stories);
    $opts['cursor'] = $result->getNextPageCursor();
    
    print_r("Fetched " . count($stories) . 
      " stories. Total story count so far: " . count($fetched_stories) . "\n");
  }
  
  return $fetched_stories;
}


// Configure API key authorization: app_id
Aylien\NewsApi\Configuration::getDefaultConfiguration()->setApiKey('X-AYLIEN-NewsAPI-Application-ID', 'YOUR_APP_ID');

// Configure API key authorization: app_key
Aylien\NewsApi\Configuration::getDefaultConfiguration()->setApiKey('X-AYLIEN-NewsAPI-Application-Key', 'YOUR_APP_KEY');

$api_instance = new Aylien\NewsApi\Api\DefaultApi();

$params = array(
  'published_at_start' => 'NOW-1HOUR',
  'published_at_end' => 'NOW',
  'language' => ['en'],
  'categories_taxonomy' => 'iab-qag',
  'categories_id' => ['IAB17'],
  'per_page' => 16
);

$stories = fetch_new_stories($params);

print_r("************\n");
print('Fetched ' . count($stories) . 
  " stories which are in English, are about Sports and were published between 1 hour ago and now\n");
import com.aylien.newsapi.ApiClient;
import com.aylien.newsapi.ApiException;
import com.aylien.newsapi.Configuration;
import com.aylien.newsapi.api.DefaultApi;
import com.aylien.newsapi.auth.ApiKeyAuth;
import com.aylien.newsapi.models.Stories;
import com.aylien.newsapi.models.Story;
import com.aylien.newsapi.parameters.StoriesParams;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class Main {
    public static void main(String[] args) throws InterruptedException {
        ApiClient defaultClient = Configuration.getDefaultApiClient();

        // Configure API key authorization: app_id
        ApiKeyAuth app_id = (ApiKeyAuth) defaultClient.getAuthentication("app_id");
        app_id.setApiKey("YOUR_APP_ID");

        // Configure API key authorization: app_key
        ApiKeyAuth app_key = (ApiKeyAuth) defaultClient.getAuthentication("app_key");
        app_key.setApiKey("YOUR_APP_KEY");

        DefaultApi apiInstance = new DefaultApi();

        StoriesParams.Builder storiesBuilder = StoriesParams.newBuilder();

        storiesBuilder.setLanguage(Arrays.asList("en"));
        storiesBuilder.setPublishedAtStart("NOW-1HOUR");
        storiesBuilder.setPublishedAtEnd("NOW");
        storiesBuilder.setCategoriesTaxonomy("iab-qag");
        storiesBuilder.setCategoriesId(Arrays.asList("IAB17"));
        storiesBuilder.setCursor("*");
        storiesBuilder.setPerPage(16);

        List<Story> stories = fetchNewStories(apiInstance, storiesBuilder);

        StoriesParams storiesParams = storiesBuilder.build();
        System.out.println("*****************");
        System.out.format("Fetched %d stories which are in English, are about Sports and were published between %s and %s\n",
                stories.size(), storiesParams.getPublishedAtStart(), storiesParams.getPublishedAtEnd());
    }

    public static List<Story> fetchNewStories(DefaultApi apiInstance, StoriesParams.Builder storiesParams) {
        List<Story> fetchedStories = new ArrayList<Story>();
        List<Story> stories = null;

        while (stories == null || stories.size() > 0) {
            try {
                Stories storiesResponse = apiInstance.listStories(storiesParams.build());
                stories = storiesResponse.getStories();
                storiesParams.setCursor(storiesResponse.getNextPageCursor());

                fetchedStories.addAll(stories);

                System.out.format("Fetched %d stories. Total story count so far: %d\n",
                        stories.size(), fetchedStories.size());
            } catch (ApiException e) {
                if (e.getCode() == 429) {
                    System.out.println("Usage limit are exceeded. Wating for 60 seconds...");
                    try {
                        Thread.sleep(1000 * 60);
                    } catch (InterruptedException ex) {
                        ex.printStackTrace();
                    }
                    continue;
                }
            }
        }

        return fetchedStories;
    }
}
# Load the gem
require 'aylien_news_api'

def fetch_new_stories(params={})
  fetched_stories = []
  stories = nil
  
  while stories.nil? || stories.size > 0
    begin
      result = $api_instance.list_stories(params)
    rescue AylienNewsApi::ApiError => e
      if e.code == 429
        puts 'Usage limit are exceeded. Wating for 30 seconds...'
        sleep(30)
        retry
      end
    end
    
    stories = result.stories
    params[:cursor] = result.next_page_cursor
    
    fetched_stories += stories
    puts "Fetched #{stories.size} stories. Total story count so far: #{fetched_stories.size}"
  end
  
  fetched_stories
end
  
# Setup authorization
AylienNewsApi.configure do |config|
  # Configure API key authorization: app_id
  config.api_key['X-AYLIEN-NewsAPI-Application-ID'] = 'YOUR_APP_ID'
  
  # Configure API key authorization: app_key
  config.api_key['X-AYLIEN-NewsAPI-Application-Key'] = 'YOUR_APP_KEY'
end

# create an instance of the API class
$api_instance = AylienNewsApi::DefaultApi.new

opts = {
  :language => ['en'],
  :published_at_start => 'NOW-1HOUR',
  :published_at_end => 'NOW',
  :categories_taxonomy => 'iab-qag',
  :categories_id => ['IAB17'],
  :cursor => '*',
  :per_page => 16
}

stories = fetch_new_stories(opts)

puts "*"*80
puts "Fetched #{stories.size} stories which are "\
  "in English, are about Sports and were published between 1 hour ago and now"
package main

// Import the library
import (
	"fmt"
	newsapi "github.com/AYLIEN/aylien_newsapi_go"
	"time"
)

func main() {
	api := newsapi.NewDefaultApi()

	// Configure API key authorization: app_id
	api.Configuration.APIKeyPrefix["X-AYLIEN-NewsAPI-Application-ID"] = "YOUR_APP_ID"

	// Configure API key authorization: app_key
	api.Configuration.APIKeyPrefix["X-AYLIEN-NewsAPI-Application-Key"] = "YOUR_APP_KEY"

	storiesParams := &newsapi.StoriesParams{
		Language:           []string{"en"},
		PublishedAtStart:   "NOW-1HOUR",
		PublishedAtEnd:     "NOW",
		CategoriesTaxonomy: "iab-qag",
		CategoriesId:       []string{"IAB17"},
		Cursor:             "*",
		PerPage:            16}

	stories := fetchNewStories(api, storiesParams)

	fmt.Println("***************")
	fmt.Printf("Fetched %d stories which are in English, are about Sports and were published between %s and %s\n",
		len(stories), storiesParams.PublishedAtStart, storiesParams.PublishedAtEnd)
}

func fetchNewStories(apiInstance *newsapi.DefaultApi, params *newsapi.StoriesParams) []newsapi.Story {
	var fetchedStories []newsapi.Story
	var stories []newsapi.Story
	isFirstCall := true

	for isFirstCall || len(stories) > 0 {
		storiesResponse, res, err := apiInstance.ListStories(params)
		if err != nil {
			panic(err)
		}

		if res.Response.StatusCode == 429 {
			fmt.Println("Usage limit are exceeded. Wating for 60 seconds...")
			time.Sleep(time.Duration(60) * time.Second)
			continue
		}

		isFirstCall = false
		stories = storiesResponse.Stories
		params.Cursor = storiesResponse.NextPageCursor

		fetchedStories = append(fetchedStories, stories...)

		fmt.Printf("Fetched %d stories. Total story count so far: %d\n",
			len(stories), len(fetchedStories))
	}

	return fetchedStories
}
using System;
using Aylien.NewsApi.Api;
using Aylien.NewsApi.Client;
using Aylien.NewsApi.Model;
using System.Collections.Generic;

namespace ResultsPagination
{
    class Program
    {
        static DefaultApi apiInstance;

        static void Main(string[] args)
        {
            // Configure API key authorization: app_id
            Configuration.Default.ApiKey.Add("X-AYLIEN-NewsAPI-Application-ID", "YOUR_APP_ID");

            // Configure API key authorization: app_key
            Configuration.Default.ApiKey.Add("X-AYLIEN-NewsAPI-Application-Key", "YOUR_APP_KEY");

            apiInstance = new DefaultApi();
            var publishedAtStart = "NOW-1HOUR";
            var publishedAtEnd = "NOW";

            var stories = FetchNewStories(publishedAtStart, publishedAtEnd);
            Console.WriteLine("*****************");
            Console.WriteLine(string.Format("Fetched {0} stories which are in English, are about Sports and were published between {1} and {2}",
                    stories.Count, publishedAtStart, publishedAtEnd));
        }

        static List<Story> FetchNewStories(String publishedAtStart, String publishedAtEnd)
        {
            var cursor = "*";
            var fetchedStories = new List<Story>();
            List<Story> stories = null;

            while (stories == null || stories.Count > 0)
            {
                try
                {
                    var storiesResponse = apiInstance.ListStories(
                        language: new List<String> { "en" },
                        publishedAtStart: publishedAtStart,
                        publishedAtEnd: publishedAtEnd,
                        categoriesTaxonomy: "iab-qag",
                        categoriesId: new List<String> { "IAB17" },
                        cursor: cursor,
                        perPage: 16
                    );

                    stories = storiesResponse._Stories;
                    cursor = storiesResponse.NextPageCursor;
                    fetchedStories.AddRange(stories);

                    Console.WriteLine(string.Format("Fetched {0} stories. Total story count so far: {1}",
                        stories.Count, fetchedStories.Count));
                }
                catch (ApiException e)
                {
                    if (e.ErrorCode == 429)
                    {
                        Console.WriteLine("Usage limit are exceeded. Wating for 60 seconds...");
                        System.Threading.Thread.Sleep(1000 * 60);
                    }
                    else
                    {
                        Console.WriteLine("Exception when calling DefaultApi.ListStories: " + e.Message);
                    }

                }
            }
            return fetchedStories;
        }
    }
}