Created
February 1, 2022 14:05
-
-
Save pvillard31/2b7e6450884e5905778565e4790e0e22 to your computer and use it in GitHub Desktop.
sidecar-example.yml for MiNiFi and NiFi
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: v1 | |
data: | |
# TODO: replace with own minifi properties | |
minifi.properties: | | |
# Core Properties # | |
nifi.version=0.11.0 | |
nifi.flow.configuration.file=./conf/config.yml | |
nifi.administrative.yield.duration=30 sec | |
# If a component has no work to do (is "bored"), how long should we wait before checking again for work? | |
nifi.bored.yield.duration=100 millis | |
# Comma separated path for the extension libraries. Relative path is relative to the minifi executable. | |
nifi.extension.path=../extensions/* | |
# Provenance Repository # | |
nifi.provenance.repository.directory.default=${MINIFI_HOME}/provenance_repository | |
nifi.provenance.repository.max.storage.time=1 MIN | |
nifi.provenance.repository.max.storage.size=1 MB | |
nifi.flowfile.repository.directory.default=${MINIFI_HOME}/flowfile_repository | |
nifi.database.content.repository.directory.default=${MINIFI_HOME}/content_repository | |
nifi.provenance.repository.class.name=NoOpRepository | |
nifi.content.repository.class.name=DatabaseContentRepository | |
#nifi.remote.input.secure=true | |
#nifi.security.need.ClientAuth= | |
#nifi.security.client.certificate= | |
#nifi.security.client.private.key= | |
#nifi.security.client.pass.phrase= | |
#nifi.security.client.ca.certificate= | |
#nifi.security.use.system.cert.store= | |
# Optional username/password used to authenticate against NiFi in RemoteProcessorGroups (i.e. Site-to-site configurations) | |
#nifi.rest.api.user.name=admin | |
#nifi.rest.api.password=password | |
# State storage configuration # | |
## The default state storage can be overridden by specifying a controller service instance | |
## that implements CoreComponentStateManagementProvider | |
## (e.g. an instance of RocksDbPersistableKeyValueStoreService or UnorderedMapPersistableKeyValueStoreService) | |
#nifi.state.management.provider.local= | |
#nifi.state.manangement.provider.local.path= | |
## To make the default state storage persist every state change, set this to true | |
## this comes at a performance penalty, but makes sure no state is lost even on unclean shutdowns | |
#nifi.state.management.provider.local.always.persist=true | |
## To change the frequency at which the default state storage is persisted, modify the following | |
#nifi.state.management.provider.local.auto.persistence.interval=1 min | |
# Disk space watchdog # | |
## Stops MiNiFi FlowController activity (excluding C2), when the available disk space on either of the repository | |
## volumes go below stop.threshold, checked every interval, then restarts when the available space on all | |
## repository volumes reach at least restart.threshold. | |
#minifi.disk.space.watchdog.enable=true | |
#minifi.disk.space.watchdog.interval=15 sec | |
#minifi.disk.space.watchdog.stop.threshold=100 MB | |
#minifi.disk.space.watchdog.restart.threshold=150 MB | |
## Enabling C2 Uncomment each of the following options | |
## define those with missing options | |
#nifi.c2.enable=true | |
## define protocol parameters | |
## The default is RESTSender. | |
## Alternatively, you may use CoapProtocol if that extension is built. | |
#nifi.c2.agent.protocol.class=RESTSender | |
#nifi.c2.agent.protocol.class=CoapProtocol | |
#nifi.c2.agent.coap.host= | |
#nifi.c2.agent.coap.port= | |
## base URL of the c2 server, | |
## very likely the same base url of rest urls | |
#nifi.c2.flow.base.url= | |
#nifi.c2.rest.url= | |
#nifi.c2.rest.url.ack= | |
nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation | |
## Minimize heartbeat payload size by excluding agent manifest from the heartbeat | |
nifi.c2.full.heartbeat=false | |
## heartbeat 4 times a second | |
#nifi.c2.agent.heartbeat.period=250 | |
## define parameters about your agent | |
#nifi.c2.agent.class= | |
#nifi.c2.agent.identifier= | |
## define metrics reported | |
nifi.c2.root.class.definitions=metrics | |
nifi.c2.root.class.definitions.metrics.name=metrics | |
nifi.c2.root.class.definitions.metrics.metrics=typedmetrics | |
nifi.c2.root.class.definitions.metrics.metrics.typedmetrics.name=RuntimeMetrics | |
nifi.c2.root.class.definitions.metrics.metrics.queuemetrics.name=QueueMetrics | |
nifi.c2.root.class.definitions.metrics.metrics.queuemetrics.classes=QueueMetrics | |
nifi.c2.root.class.definitions.metrics.metrics.typedmetrics.classes=ProcessMetrics,SystemInformation | |
nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.name=ProcessorMetric | |
nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.classes=GetFileMetrics | |
## Specify multiple heartbeat handlers. | |
## HeartbeatLogger logs the heartbeats on TRACE for debugging. | |
#nifi.c2.agent.heartbeat.reporter.classes=HeartbeatLogger | |
## enable the controller socket provider on port 9998 | |
## off by default. C2 must be enabled to support these | |
#controller.socket.host=localhost | |
#controller.socket.port=9998 | |
#JNI properties | |
nifi.framework.dir=${MINIFI_HOME}/minifi-jni/lib | |
nifi.nar.directory=${MINIFI_HOME}/minifi-jni/nars | |
nifi.nar.deploy.directory=${MINIFI_HOME}/minifi-jni/nardeploy | |
nifi.nar.docs.directory=${MINIFI_HOME}/minifi-jni/nardocs | |
# must be comma separated | |
nifi.jvm.options=-Xmx1G | |
nifi.python.processor.dir=${MINIFI_HOME}/minifi-python/ | |
config.yml: | | |
Flow Controller: | |
name: minifi-logging | |
Processors: | |
- id: 94b8e610-b4ed-1ec9-b26f-c839931bf3e2 | |
name: TailFile | |
class: org.apache.nifi.processors.standard.TailFile | |
max concurrent tasks: 1 | |
scheduling strategy: TIMER_DRIVEN | |
scheduling period: 5 sec | |
auto-terminated relationships list: [] | |
Properties: | |
File to Tail: /nifi-logs/nifi-app.log | |
tail-mode: Single file | |
Lookup frequency: 1 min | |
- id: 261e8cf1-71ba-af86-fb2b-bc95764f91f8 | |
name: MergeContent | |
class: org.apache.nifi.processors.standard.MergeContent | |
max concurrent tasks: 1 | |
scheduling strategy: EVENT_DRIVEN | |
auto-terminated relationships list: | |
- original | |
Properties: | |
# TODO optmize merging strategy for your own preferences | |
Attribute Strategy: Keep Only Common Attributes | |
Maximum number of Bins: 100 | |
Minimum Group Size: 0 | |
Max Bin Age: 1 min | |
Minimum Number of Entries: 1000000 | |
Maximum Group Size: 1000000 | |
Maximum Number of Entries: 1000000 | |
Merge Strategy: Bin-Packing Algorithm | |
- id: 69335770-ee29-11eb-9a03-0242ac130003 | |
name: CompressContent | |
class: org.apache.nifi.processors.standard.CompressContent | |
max concurrent tasks: 1 | |
scheduling strategy: EVENT_DRIVEN | |
auto-terminated relationships list: | |
- failure | |
Properties: | |
Compression Level: 6 | |
Compression Format: gzip | |
UpdateFileName: false | |
- id: fe198bd9-2a1c-316e-0000-000000000000 | |
name: PutS3Object | |
class: org.apache.nifi.minifi.azure.processors.PutS3Object | |
max concurrent tasks: 1 | |
scheduling strategy: EVENT_DRIVEN | |
auto-terminated relationships list: | |
- success | |
Properties: | |
Bucket: test_bucket # TODO: replace with own S3 bucket name | |
AWS Credentials Provider service: AWSCredentialsService | |
Controller Services: | |
- name: AWSCredentialsService | |
id: 2094d776-2006-4d02-9bb9-28eac9d0fc95 | |
class: org.apache.nifi.minifi.aws.controllers.AWSCredentialsService | |
Properties: | |
Access Key: access_key # TODO: Replace with own AWS access key id | |
Secret Key: secret_key # TODO: Replace with own AWS secret key | |
Connections: | |
- id: 99f617e7-49a1-6078-8534-26af7d56ca08 | |
name: TailFile/success/MergeContent | |
source name: TailFile | |
source relationship names: | |
- success | |
destination name: MergeContent | |
max work queue size: 1000000 | |
max work queue data size: 10 MB | |
flowfile expiration: 0 sec | |
queue prioritizer class: '' | |
- id: 24d6be1e-ee29-11eb-9a03-0242ac130003 | |
name: MergeContent/merged/CompressContent | |
source name: MergeContent | |
source relationship names: | |
- merged | |
destination name: CompressContent | |
max work queue size: 1000000 | |
max work queue data size: 10 MB | |
flowfile expiration: 0 sec | |
queue prioritizer class: '' | |
- id: 67ea5c91-446a-393b-6274-b6fae2f475a2 | |
name: CompressContent/success/PutS3Object | |
source name: CompressContent | |
source relationship names: | |
- success | |
destination name: PutS3Object | |
max work queue size: 1000000 | |
max work queue data size: 10 MB | |
flowfile expiration: 0 sec | |
queue prioritizer class: '' | |
Remote Process Groups: [] | |
kind: ConfigMap | |
metadata: | |
labels: | |
k8s-app: minifi-log-collection | |
name: minifi-log-collection-config | |
namespace: default | |
--- | |
apiVersion: v1 | |
kind: Pod | |
metadata: | |
name: log-collection-minifi-pod | |
namespace: default | |
spec: | |
containers: | |
- name: nifi | |
image: apache/nifi:latest | |
volumeMounts: | |
- name: nifi-logs | |
mountPath: /opt/nifi/nifi-current/logs | |
- name: sidecar-minifi | |
image: container.repo.cloudera.com/cloudera/apacheminificpp:latest | |
volumeMounts: | |
- name: nifi-logs | |
mountPath: /nifi-logs | |
- name: minificonfig | |
mountPath: /opt/minifi/minifi-current/conf/config.yml | |
subPath: config.yml | |
- name: minificonfig | |
mountPath: /opt/minifi/minifi-current/conf/minifi.properties | |
subPath: minifi.properties | |
volumes: | |
- name: nifi-logs | |
emptyDir: {} | |
- configMap: | |
defaultMode: 420 | |
name: minifi-log-collection-config | |
name: minificonfig |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment