-
Step 1 (ulimit): open the sysctl.conf and add this line fs.file-max = 65536
vi /etc/sysctl.conf
add following at end of file in above file:
fs.file-max = 65536
save and exit.
from billiard.context import Process | |
from scrapy.crawler import Crawler | |
from scrapy import signals | |
from scrapy.utils.project import get_project_settings | |
from twisted.internet import reactor | |
from celery_app import app | |
class CrawlerProcess(Process): |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: elasticsearch-logging | |
namespace: kube-system | |
labels: | |
k8s-app: elasticsearch-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile |
# maximum capability of system | |
user@ubuntu:~$ cat /proc/sys/fs/file-max | |
708444 | |
# available limit | |
user@ubuntu:~$ ulimit -n | |
1024 | |
# To increase the available limit to say 200000 | |
user@ubuntu:~$ sudo vim /etc/sysctl.conf |
import boto3 | |
import json | |
# update these with your actual settings | |
maxreceivecount = 3 | |
queueurl = "https://sqs.region.amazonaws.com/accountnumber/queuename" | |
deadlettertargetarn = "arn:aws:sqs:region:accountnumber:deadletterqueuename" | |
# you could just create a json string | |
policy = {"maxReceiveCount" : maxreceivecount, "deadLetterTargetArn": deadlettertargetarn} |
# This is a quickstart! In the real world use a real broker (message queue) | |
# such as Redis or RabbitMQ !! | |
BROKER_URL = 'sqlalchemy+sqlite:///tasks.db' | |
CELERY_RESULT_BACKEND = "db+sqlite:///results.db" | |
CELERY_IMPORTS = ['tasks'] | |
CELERY_TASK_SERIALIZER = 'json' | |
CELERY_RESULT_SERIALIZER = 'json' |
'use strict'; | |
/** | |
* Implementing main examples from Martin Fowler article | |
* on Collection Pipeline programming pattern: | |
* http://martinfowler.com/articles/collection-pipeline/ | |
* with #javascript and underscore.js: http://underscorejs.org/ | |
* | |
* tested with nodejs v0.10.32 | |
* | |
* underscore.js is required: |
"""An example of how to perform a multi-threaded unit test of a web service. | |
The particulars of this example make use of some conventions used when | |
developing a web service when using the Flask microframework, but can be | |
modified to fit most needs. | |
""" | |
import json | |
import threading | |
import time |
A warning occurred (42 apples) | |
An error occurred |