Problems with the Pagination using HTTPS in Node.JS

Hi guys!

So i am currently writing a discord bot. This discord bot is supposed to return all the streams for a certain game. However, since the Helix API allows a maximum 100 results per page and uses pagination to “tab through”, i am kind of facing a problem here. This is what my code currently looks like (the relevant part of it):

doGetRequest = (reqpath, callback) => {
    var options = {
      host: `api.twitch.tv`,
      port: 443,
      path: reqpath,
      method: 'GET',
      headers: { 'Client-Id': twitchid, 'Authorization': `Bearer ${twitchtoken}` },
    };
  
    https.get(options, response => {
      let data = "";
      response.on("data", d => data += d);
      response.on("end", () => callback(data));
    });
  }

var data_full = {}
doGetRequest('/helix/streams?game_id=509658', response => {
    parsed = JSON.parse(response);
    data_full = parsed.data;
    while(parsed.pagination.cursor) {
        doGetRequest(`/helix/streams?game_id=509658&after=${parsed.pagination.cursor}`, res => {
            parsed = JSON.parse(res);
            data_full += parsed.data;
        })
    }
});

I think the problem is that the parsed variable is outside of the scope of the inner get request.

Can someone help me fix this pls ?:confused:

Thanks so much!

Heres what I do in plain JS (which would carry over to nodeJS

        var limit = 50;
        function fetchStreams(game_id, page, after, tot) {
            page = page ? page : 0;
            document.getElementById('loader_' + game_id).textContent = 'Loading Page: ' + page + ' Current ' + tot;
            fetch(
                'https://api.twitch.tv/helix/streams?first=100&game_id=' + game_id + (after ? '&after=' + after : ''),
                {
                    "headers": {
                        "Client-ID": client_id,
                        "Authorization": "Bearer " + access_token
                    }
                }
            )
            .then(resp => resp.json())
            .then(resp => {
                document.getElementById('loader_' + game_id).textContent = 'Processing Page: ' + page;

                var total = parseInt(document.getElementById('count_' + game_id).textContent);
                for (var x=0;x<resp.data.length;x++) {
                    total += resp.data[x].viewer_count;
                }
                document.getElementById('count_' + game_id).textContent = total;

                var d = document.createElement('td');
                d.textContent = resp.data.length;
                document.getElementById('loader_row_' + game_id).append(d);

                // loop if we got a cursor
                if (resp.hasOwnProperty('pagination') && resp.pagination.hasOwnProperty('cursor')) {
                    page++;
                    // do a page limit check
                    if (page >= limit) {
                        document.getElementById('loader_' + game_id).textContent = 'Stopped at Page: ' + page + ' - ' + resp.data.length;
                        return;
                    }
                    fetchStreams(game_id, page, resp.pagination.cursor, total);
                } else {
                    document.getElementById('loader_' + game_id).textContent = 'Last Page: ' + page + ' - ' + resp.data.length;
                }
            })
            .catch(err => {
                console.log(err);
                document.getElementById('loading').textContent = 'Something went wrong';
            });
        }
  • Call fetchStreams(game_id)
  • If response contains a cursor then call fetchStreams(game_id, theCursor)
  • if response contains no cursor (or hit max page limiter - I stop at 50 here) call a completion function

This example exists as a self runnable browser example

https://barrycarlyon.github.io/twitch_misc/examples/browse_categories/

Hey man, thanks for the quick reply!

The problem is, that i want to store the data of all the pages in a variable to further process them.

Or am i missing something here?

Then do that instead of writing to the DOM (like my example does)

Just wanted to illistrate an example of how I paginate.

So you probably want something like

Pseudo javascript

let streams = [];

function getStreams(cursor) {
    let url = new URL('https://api.twitch.tv/helix/streams');
    let params = [
        [ 'first', '100' ],
        [ 'game_id', '509658' ]
    ];
    if (cursor) {
        params.push([ 'after', cursor ]);
    }
    url.search = new URLSearchParams(params).toString();
    // make request to twitch using `url`

    // Parse response

    // store streams in the array
    streams = streams.concat(resp.data);
    // another page?
    if (resp.hasOwnProperty('pagination') && resp.pagination.hasOwnProperty('cursor')) {
        getStreams(resp.pagination.cursor);
    } else {
        // all done
        processStreams();
   }
}

function processStreams() {
    // do something with `streams`
}

So essentially i would like to sum up the viewers of all the streams in that specific game.
However, if i process them in a function, i can still only work with each set of 100 streams right?

    // store streams in the array
    streams = streams.concat(resp.data);

Will put all the streams into the streams array

But if you only want the total viewers.

var total_viewers = 0;

function getStreams(cursor) {
    // snip do request
    resp.data.forEach(stream => {
        total_viewers += stream.viewer_count
    });
    // snip do paginate
}

100 per page the code checks for a cursor if it has a cursor it self calls the getStreams with the cursor

let streams = []; is hoisted outside the function as a global so would collect all streams into that array. Each call merging the results from the call into that array

That makes a lot of sense.

The thing is, if i have 20 games and i want to fetch the viewer count for all of them and count the number of streams per game, that wouldnt work anymore, would it?

Also, doesn’t let declare scoped vars? sorry i’m a bit confused

Then


let total_viewers = [];

function getStreams(game_id, cursor) {
    // snip do request
    resp.data.forEach(stream => {
        if (!total_viewers.hasOwnProperty(game_id)) {
            total_viewers[game_id] = 0;
        }
        total_viewers[game_id] += stream.viewer_count
    });
    // snip do paginate
}

It stops the variable being used in the same scope.

So I can’t have two total_viewers in the global scope
But I could have a let total_viewers inside the function scoped to the function, but since I didn’t the global scoped one is the one affected.

So essentially the global let tells the “total_viewers” inside the function “Nah G, you exist already”?

No

this is valid


let variable = 'wotsit';

function thing() {
    let variable = 'somethingelse';
    console.log('Function is', variable);
}

console.log('Outside of function', variable);
thing();
console.log('Outside of function', variable);

result

Outside of function wotsit
Function is somethingelse
Outside of function wotsit

image

Doesn’t matter what i try, the “parsed” var (see original post) will always have the same value… i tried it with the “let” in different positions… doesnt change

My propsal is to rewrite your code from scratch.

Since your code doesn’t seem to wait for doGetRequest to run/complete

So you cause an infinity loop. and run out of memory/crash the script.

doGetRequest
while(parsed.pagination.cursor) {
   doGetRequest(nextpage, {
        updatePartse
   });
})

Results in

doGetRequest for page zero
doGetRequest for page one
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
recieve page 1
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
doGetRequest(page1);
page2processed
doGetRequest(page2);
doGetRequest(page2);
doGetRequest(page2);
doGetRequest(page2);
doGetRequest(page2);
doGetRequest(page2);
doGetRequest(page2);
doGetRequest(page2);
doGetRequest(page2);
Memeory leak and crash

while(parsed.pagination.cursor) { the vlaud in parsed isn’t updated quick enough cause page1 calls to loop whilst page2 call is being made.

This topic was automatically closed 30 days after the last reply. New replies are no longer allowed.