yarikc

version: '2'
services:
  
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:5.3.0
    container_name: elasticsearch
    environment:
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"

yarikc

import traceback

import couchdb

couch = couchdb.Server('http://admin:admin@127.0.0.1:5984/')

try:
    db = couch.create('test1')
except Exception, e:
    traceback.print_exc()

yarikc

Tuning Storm+Trident

Tuning a dataflow system is easy:

The First Rule of Dataflow Tuning:
* Ensure each stage is always ready to accept records, and
* Deliver each processed record promptly to its destination

yarikc

from bs4 import BeautifulSoup
import requests
import urllib
import gzip
import os
import time
import re

url_base = 'http://www1.ncdc.noaa.gov/pub/data/noaa/{}'
ext = 'gz'

yarikc

import com.google.common.base.Stopwatch;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import java.util.Random;
import java.util.concurrent.TimeUnit;


public class Log4jRecordGenerator {
    public static void main(String[] args) {

yarikc

# reading log4j records with layout %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}\u001F%p\u001F%m%n
# need to use message generator that adds sequence number to position 2

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.topology.base.BaseBasicBolt;

yarikc

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /var/log/example.log
a1.sources.r1.channels = c1

a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink