config map ingestor

config.yaml

resource "kubernetes_config_map" "ingestor" {
  depends_on = ["kubernetes_config_map.aws_auth"]

  metadata {
    name      = "fqe-ingestor-config"
    namespace = "default"
  }

  data {
    config.cfg = <<CONFIG
[sentry]
token = '50f7646f38834025a9ab8aca5cdfd98b:239eead97e924364a9984b4f5315b03c'
project = '1242164'

[diffbot]
batch_size = 2000
poll_delay = 2
notify_email = 'dev+null@factmata.com'
token = '20c381720a574ab693e55b93f9b71b39'
slack_url = 'https://hooks.slack.com/services/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'

[queue]
[[analysis]]
name = 'pre-score'
connection = 'amqp://ingestor:${random_string.ingestor.result}@rabbit-rabbitmq-ha/pre-score'

[[fmindex]]
name = 'fmindex-writer'
connection = 'amqp://distributor:${random_string.distributor.result}@rabbit-rabbitmq-ha/fmindex'

[[ingestor-in]]
name = 'ingestor-in'
connection = 'amqp://ingestor:${random_string.ingestor.result}@rabbit-rabbitmq-ha/ingestor-in'

[sovrn]
ingestor_behaviour = 'poll'
ingest_type= 's3'
egest_type = 'jsonl'
aws_profile = 'pipeline-ingestor'
in_bucket_name = 'fqe-ingestion-in-staging'
out_bucket_name = 'fqe-ingestion-ack-staging'
file_prefix = 'scraped_text'
file_suffix = ''
aws_region_name = 'eu-west-1'
write_to_job_store = 'True'
write_to_index = 'True'

[[models]]
HATE=3
BIAS=3

[factmata_noscrape]
ingestor_behaviour = 'poll'
ingest_type = 's3'
egest_type = 'jsonl'
aws_profile = 'pipeline-ingestor'
in_bucket_name = 'fqe-ingestion-in-staging'
out_bucket_name = 'fqe-ingestion-ack-staging'
file_prefix = 'factmata_noscrape'
file_suffix = ''
aws_region_name = 'eu-west-1'
write_to_job_store = 'False'
write_to_index = 'True'

[[models]]
HATE=3
BIAS=3

[factmata_scrape]
ingestor_behaviour = 'poll'
ingest_type = 's3'
egest_type = 'jsonl_only_url'
aws_profile = 'pipeline-ingestor'
in_bucket_name = 'fqe-ingestion-in-staging'
out_bucket_name = 'fqe-ingestion-ack-staging'
file_prefix = 'factmata_scrape'
file_suffix = ''
aws_region_name = 'eu-west-1'
result_bucket_name='fqe-scrape-results-staging'
scraper='diffbot'
write_to_job_store = 'False'
write_to_index = 'True'

[[models]]
HATE=3
BIAS=3

[trendolizer]
ingestor_behaviour = 'poll'
ingest_type = 's3'
egest_type = 'trendolizer'
aws_profile = 'pipeline-ingestor'
in_bucket_name = 'fqe-ingestion-in-staging'
out_bucket_name = 'fqe-scrape-ack-staging'
file_prefix = 'trendolizer'
file_suffix = ''
aws_region_name = 'eu-west-1'
result_bucket_name = 'fqe-scrape-results-staging'
scraper = 'diffbot'
write_to_job_store = 'False'
write_to_index = 'True'

[[models]]
HATE=3
BIAS=3

[krzana]
ingestor_behaviour = 'poll'
ingest_type = 's3'
egest_type = 'krzana'
aws_profile = 'pipeline-ingestor'
in_bucket_name = 'fqe-scrape-in-staging'
out_bucket_name = 'fqe-scrape-ack-staging'
file_prefix = 'krzana'
file_suffix = ''
aws_region_name = 'eu-west-1'
result_bucket_name = 'fqe-scrape-results-staging'
scraper = 'diffbot'
write_to_job_store = False
write_to_index = True

[[models]]
HATE=3
BIAS=3

[local]
ingestor_behaviour = 'poll'
ingest_type = 'local'
egest_type = 'jsonl'
dir = '/app/config'
file_pattern = '*.jsonl'
completed_suffix = '.done'

[[models]]
HATE=3
BIAS=3

[queue_interface]
ingestor_behaviour = 'real_time'
ingest_type = 'queue'
interface_name = 'ingestor-in'
egest_type = 'json_flexible'
scraper = 'diffbot'
write_to_job_store = False
write_to_index = True

[[models]]
HATE=3
BIAS=3
CONFIG
  }