ianrussel · July 26, 2024 14:02
diff --git a/gistfile1.txt b/gistfile1.txt
 def _request_product(asin):
    print(f" asin {asin}")
    retry_count = 0
    max_retries = 4

    while retry_count < max_retries:
        try:
            products = api.query(asin, domain='GB', rating=1, buybox=1, update=0)
            break  # If the request is successful, break out of the loop
        except ReadTimeout:
            retry_count += 1
            print(f"Request timed out for ASIN {asin}. Retrying {retry_count}/{max_retries}...")
            if retry_count == max_retries:
                print("Max retries reached. Moving on to the next task.")
                return
        except Exception as e:
            print(f"An error occurred while querying the API: {e}")
            return

    # print(f" rating {products[0]['data']['RATING']}")
    with PostgreSQLConnector() as db_connector:
        try:
            newpricetime = products[0]['data']['NEW_time']
            price = get_price(products[0]['csv'][18])
            stock = float(products[0]['numberOfItems'])
            stock = max(0, round(stock))
            final_rating = None
            final_review = None
            if 'RATING' in products[0]['data']:
                rating = products[0]['data']['RATING']
                final_rating = round(float(rating[-1]), 1)
            if 'COUNT_REVIEWS' in products[0]['data']:
                reviews = products[0]['data']['COUNT_REVIEWS']
                final_review = float(reviews[-1])
            print(f" price {price} asin {asin} stock {products[0]['numberOfItems']} stock rounded {stock}")
            in_stock_status = 0
            if products[0]['availabilityAmazon'] in [0, 4]:
                in_stock_status = 1
            amazon_object = db_connector.session.query(Amazon).filter(Amazon.asin == asin).one()
            if amazon_object:
                amazon_object.price = price
                amazon_object.stock = stock
                amazon_object.in_stock_status = in_stock_status
                amazon_object.customer_reviews = final_review
                amazon_object.total_no_of_ratings = final_rating
                amazon_object.last_update = newpricetime[-1].strftime("%Y/%m/%d %H:%M:%S")
                db_connector.session.commit()
            else:
                print("Amazon object with the given ASIN not found")
        except NoResultFound:
            print("Amazon object with the given ASIN not found")
            db_connector.session.rollback()
        except Exception as e:
            print(f"An error occurred: {e}")
            db_connector.session.rollback()
        finally:
            db_connector.session.close()


 def run_main():
    token = get_token_status()
    usable_tokens = float(token['tokensLeft']) - float(keep_tokens)
    if float(usable_tokens) <= float(keep_tokens):
        # raise Exception("Not enough tokens left")
        print(f" keep tokens {float(keep_tokens)} usable tokens {float(usable_tokens)} Not enough tokens left")
        time.sleep(60)
    limit = math.floor(usable_tokens)
    print(f" limit {limit} usable tokens  {usable_tokens}")
    with PostgreSQLConnector() as db_connector:
        try:
            asins = db_connector.session.query(Amazon.asin) \
                .join(Product, Product.asin == Amazon.asin) \
                .filter(Amazon.asin is not None, Amazon.asin != '', Amazon.is_linked == 1) \
                .filter(Product.status == 1, Product.master != 2, Product.is_category_hidden != 1) \
                .filter(or_(Product.primary_category_path.like('%gaming%'), Product.primary_category_path.like('%computing%'))).order_by(  # noqa
                    case(
                        (Amazon.modified_at == None, 1),  # noqa: E711
                        (text(
                            "NOT (CAST(amazon.modified_at AS TEXT) ~ '^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}\\+\\d{2}$')"), 1),  # noqa
                        else_=2
                    ),
                    Amazon.modified_at.asc()
                ).limit(200)

            # Convert result to a list of ASIN strings
            asin_list = [asin[0] for asin in asins]

            print(f"Total ASINs to process: {len(asin_list)})")

            # Processing ASINs in batches
            batch_size = 60
            num_batches = len(asin_list) // batch_size + (1 if len(asin_list) % batch_size > 0 else 0)

            for i in range(num_batches):
                start_index = i * batch_size
                end_index = start_index + batch_size
                batch = asin_list[start_index:end_index]

                for asin in batch:
                    print(f"Processing ASIN: {asin}")
                    _request_product(asin)

                # Sleep for 2 seconds after processing each batch
                time.sleep(1)

                # Check if it's not the last batch before sleeping for a minute
                if i < num_batches - 1:
                    print("Waiting for 10 seconds before processing the next batch...")
                    time.sleep(10)
        except Exception:
            db_connector.session.rollback()
        finally:
            db_connector.session.close()
	def _request_product(asin):
	print(f" asin {asin}")
	retry_count = 0
	max_retries = 4

	while retry_count < max_retries:
	try:
	products = api.query(asin, domain='GB', rating=1, buybox=1, update=0)
	break # If the request is successful, break out of the loop
	except ReadTimeout:
	retry_count += 1
	print(f"Request timed out for ASIN {asin}. Retrying {retry_count}/{max_retries}...")
	if retry_count == max_retries:
	print("Max retries reached. Moving on to the next task.")
	return
	except Exception as e:
	print(f"An error occurred while querying the API: {e}")
	return

	# print(f" rating {products[0]['data']['RATING']}")
	with PostgreSQLConnector() as db_connector:
	try:
	newpricetime = products[0]['data']['NEW_time']
	price = get_price(products[0]['csv'][18])
	stock = float(products[0]['numberOfItems'])
	stock = max(0, round(stock))
	final_rating = None
	final_review = None
	if 'RATING' in products[0]['data']:
	rating = products[0]['data']['RATING']
	final_rating = round(float(rating[-1]), 1)
	if 'COUNT_REVIEWS' in products[0]['data']:
	reviews = products[0]['data']['COUNT_REVIEWS']
	final_review = float(reviews[-1])
	print(f" price {price} asin {asin} stock {products[0]['numberOfItems']} stock rounded {stock}")
	in_stock_status = 0
	if products[0]['availabilityAmazon'] in [0, 4]:
	in_stock_status = 1
	amazon_object = db_connector.session.query(Amazon).filter(Amazon.asin == asin).one()
	if amazon_object:
	amazon_object.price = price
	amazon_object.stock = stock
	amazon_object.in_stock_status = in_stock_status
	amazon_object.customer_reviews = final_review
	amazon_object.total_no_of_ratings = final_rating
	amazon_object.last_update = newpricetime[-1].strftime("%Y/%m/%d %H:%M:%S")
	db_connector.session.commit()
	else:
	print("Amazon object with the given ASIN not found")
	except NoResultFound:
	print("Amazon object with the given ASIN not found")
	db_connector.session.rollback()
	except Exception as e:
	print(f"An error occurred: {e}")
	db_connector.session.rollback()
	finally:
	db_connector.session.close()


	def run_main():
	token = get_token_status()
	usable_tokens = float(token['tokensLeft']) - float(keep_tokens)
	if float(usable_tokens) <= float(keep_tokens):
	# raise Exception("Not enough tokens left")
	print(f" keep tokens {float(keep_tokens)} usable tokens {float(usable_tokens)} Not enough tokens left")
	time.sleep(60)
	limit = math.floor(usable_tokens)
	print(f" limit {limit} usable tokens {usable_tokens}")
	with PostgreSQLConnector() as db_connector:
	try:
	asins = db_connector.session.query(Amazon.asin) \
	.join(Product, Product.asin == Amazon.asin) \
	.filter(Amazon.asin is not None, Amazon.asin != '', Amazon.is_linked == 1) \
	.filter(Product.status == 1, Product.master != 2, Product.is_category_hidden != 1) \
	.filter(or_(Product.primary_category_path.like('%gaming%'), Product.primary_category_path.like('%computing%'))).order_by( # noqa
	case(
	(Amazon.modified_at == None, 1), # noqa: E711
	(text(
	"NOT (CAST(amazon.modified_at AS TEXT) ~ '^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}\\+\\d{2}$')"), 1), # noqa
	else_=2
	),
	Amazon.modified_at.asc()
	).limit(200)

	# Convert result to a list of ASIN strings
	asin_list = [asin[0] for asin in asins]

	print(f"Total ASINs to process: {len(asin_list)})")

	# Processing ASINs in batches
	batch_size = 60
	num_batches = len(asin_list) // batch_size + (1 if len(asin_list) % batch_size > 0 else 0)

	for i in range(num_batches):
	start_index = i * batch_size
	end_index = start_index + batch_size
	batch = asin_list[start_index:end_index]

	for asin in batch:
	print(f"Processing ASIN: {asin}")
	_request_product(asin)

	# Sleep for 2 seconds after processing each batch
	time.sleep(1)

	# Check if it's not the last batch before sleeping for a minute
	if i < num_batches - 1:
	print("Waiting for 10 seconds before processing the next batch...")
	time.sleep(10)
	except Exception:
	db_connector.session.rollback()
	finally:
	db_connector.session.close()