Node JS request not keeping current session/cookies
I've been experimenting using Node JS for webscraping using request-promise, but recently I've found that when trying to create a request session by using the same cookie jar for each request, the request session does not persist. Here is my code so far:
var rp = require('request-promise'); //requests module
var jssoup = require('jssoup').default; //crawler
global.useragent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)';
function kithCheckoutLink(bypass, link){
var cookiejar = rp.jar()
if (bypass == 1){
global.options = {
uri: 'https://kith.com/cart/18914757476421:1',
jar: cookiejar,
followAllRedirects: true,
headers:{
'User-Agent':useragent
},
}
console.log(options.uri);
bypassrequest = rp.get(options, function(error, response){
var bypasslink = bypassrequest.uri.href;
console.log(bypasslink);
options.uri = 'https://kith.com/cart/change?line=1&quantity=0';
removerequest = rp.get(options, function(error, response){
console.log(removerequest.uri.href);
});
});
}
};
kithCheckoutLink(1, 'www.google.com');
So when I run the code, everything functions normally until the second GET request. I get a huge "bad request" error saying "cannot update empty cart". The thing is, with the first GET request I successfully added a product to my cart on the website and the console logs the checkout link to prove it. I've even tried this manually using my chrome browser and did not have any problems with clearing the cart. This means that the problem must be that the cookies are not transferring to each request, as if the cookies were transferring I wouldn't get an error that my cart was already empty beforehand since I just added the product to cart. What am I doing wrong here?
node.js cookies web-scraping request request-promise
asked Jan 1 at 19:37
EpacityEpacity
309
add a comment |
I've been experimenting using Node JS for webscraping using request-promise, but recently I've found that when trying to create a request session by using the same cookie jar for each request, the request session does not persist. Here is my code so far:
var rp = require('request-promise'); //requests module
var jssoup = require('jssoup').default; //crawler
global.useragent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)';
function kithCheckoutLink(bypass, link){
var cookiejar = rp.jar()
if (bypass == 1){
global.options = {
uri: 'https://kith.com/cart/18914757476421:1',
jar: cookiejar,
followAllRedirects: true,
headers:{
'User-Agent':useragent
},
}
console.log(options.uri);
bypassrequest = rp.get(options, function(error, response){
var bypasslink = bypassrequest.uri.href;
console.log(bypasslink);
options.uri = 'https://kith.com/cart/change?line=1&quantity=0';
removerequest = rp.get(options, function(error, response){
console.log(removerequest.uri.href);
});
});
}
};
kithCheckoutLink(1, 'www.google.com');
So when I run the code, everything functions normally until the second GET request. I get a huge "bad request" error saying "cannot update empty cart". The thing is, with the first GET request I successfully added a product to my cart on the website and the console logs the checkout link to prove it. I've even tried this manually using my chrome browser and did not have any problems with clearing the cart. This means that the problem must be that the cookies are not transferring to each request, as if the cookies were transferring I wouldn't get an error that my cart was already empty beforehand since I just added the product to cart. What am I doing wrong here?
node.js cookies web-scraping request request-promise
asked Jan 1 at 19:37
EpacityEpacity
309
add a comment |
I've been experimenting using Node JS for webscraping using request-promise, but recently I've found that when trying to create a request session by using the same cookie jar for each request, the request session does not persist. Here is my code so far:
var rp = require('request-promise'); //requests module
var jssoup = require('jssoup').default; //crawler
global.useragent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)';
function kithCheckoutLink(bypass, link){
var cookiejar = rp.jar()
if (bypass == 1){
global.options = {
uri: 'https://kith.com/cart/18914757476421:1',
jar: cookiejar,
followAllRedirects: true,
headers:{
'User-Agent':useragent
},
}
console.log(options.uri);
bypassrequest = rp.get(options, function(error, response){
var bypasslink = bypassrequest.uri.href;
console.log(bypasslink);
options.uri = 'https://kith.com/cart/change?line=1&quantity=0';
removerequest = rp.get(options, function(error, response){
console.log(removerequest.uri.href);
});
});
}
};
kithCheckoutLink(1, 'www.google.com');
So when I run the code, everything functions normally until the second GET request. I get a huge "bad request" error saying "cannot update empty cart". The thing is, with the first GET request I successfully added a product to my cart on the website and the console logs the checkout link to prove it. I've even tried this manually using my chrome browser and did not have any problems with clearing the cart. This means that the problem must be that the cookies are not transferring to each request, as if the cookies were transferring I wouldn't get an error that my cart was already empty beforehand since I just added the product to cart. What am I doing wrong here?
node.js cookies web-scraping request request-promise
asked Jan 1 at 19:37
EpacityEpacity
309
I've been experimenting using Node JS for webscraping using request-promise, but recently I've found that when trying to create a request session by using the same cookie jar for each request, the request session does not persist. Here is my code so far:
var rp = require('request-promise'); //requests module
var jssoup = require('jssoup').default; //crawler
global.useragent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)';
function kithCheckoutLink(bypass, link){
var cookiejar = rp.jar()
if (bypass == 1){
global.options = {
uri: 'https://kith.com/cart/18914757476421:1',
jar: cookiejar,
followAllRedirects: true,
headers:{
'User-Agent':useragent
},
}
console.log(options.uri);
bypassrequest = rp.get(options, function(error, response){
var bypasslink = bypassrequest.uri.href;
console.log(bypasslink);
options.uri = 'https://kith.com/cart/change?line=1&quantity=0';
removerequest = rp.get(options, function(error, response){
console.log(removerequest.uri.href);
});
});
}
};
kithCheckoutLink(1, 'www.google.com');
So when I run the code, everything functions normally until the second GET request. I get a huge "bad request" error saying "cannot update empty cart". The thing is, with the first GET request I successfully added a product to my cart on the website and the console logs the checkout link to prove it. I've even tried this manually using my chrome browser and did not have any problems with clearing the cart. This means that the problem must be that the cookies are not transferring to each request, as if the cookies were transferring I wouldn't get an error that my cart was already empty beforehand since I just added the product to cart. What am I doing wrong here?
node.js cookies web-scraping request request-promise
node.js cookies web-scraping request request-promise
asked Jan 1 at 19:37
EpacityEpacity
309
asked Jan 1 at 19:37
EpacityEpacity
309
asked Jan 1 at 19:37
EpacityEpacity
309
asked Jan 1 at 19:37
EpacityEpacity
309
asked Jan 1 at 19:37
EpacityEpacity
309
309
add a comment |
add a comment |
0
active
oldest
votes
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53998374%2fnode-js-request-not-keeping-current-session-cookies%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
0
active
oldest
votes
0
active
oldest
votes
active
oldest
votes
active
oldest
votes
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53998374%2fnode-js-request-not-keeping-current-session-cookies%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
