-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.cfg
155 lines (140 loc) · 4.29 KB
/
app.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
##### WRANGLER DEPLOYMENT SETTINGS #####
#
# should be set to suppport the class libraries and the application
# to use the deplyment specific paramters for the wrangler work loads
#
# See the respective sections: Implementation, Security, Database, Spark
# Timezon, Currency
#
# author: <[email protected]>
#
#####
[MODULES]
#---set of modules to include or exclude to disable simply comment # the irrelevant ones.
# give a comma separated of list the packages (or submodules) belonging to the modules
# e.g., myModule = package1, package2, ..., package n
etl= loader,transform,extractor
lib= spark,basemap,utils
ml = cluster,dimreduc,natlang,statistics,timeseries
gis= postgis
io = importer,exporter
[DATASTORE]
#---set the default data storage mode; i.e. local or cloud file system.
# e.g., mode = AWS-S3-BUCKET & root = rezaware-wrangler-source-code (bucket name)
# e.g., mode = LOCAL-FS & root = rezaware (directory name rezaware app is in)
mode = local-fs
root = /home/nuwan/workspace/HERO/
[APP]
#[APPOWNER]
#--organization name
# default: RezAWARE (rezgateway)
orgname = rezgate
#--org url
# default: https://rezgateway.com
orgurl = https://rezgateway.com
#--data hosting root location or S3 Bucket/Object
# default ../data/
orgemail = [email protected]
hostip = 127.0.0.1
datadir = ../data/
[HOSTS]
hostip = 127.0.0.1
datadir = ../data/
[SPARK]
#--settings to connect to the database to perform work loads '''
# install and setup spark: https://computingforgeeks.com/how-to-install-apache-spark-on-ubuntu-debian/
# also install findspark by running >>> python3 -m pip install findspark
# to download Postgres JDBC drivers: https://jdbc.postgresql.org/
homedir = /opt/spark_hadoop_3/
bindir = /opt/spark_hadoop_3/bin
config = spark.jars
#jardir = /opt/spark_hadoop_3/jars/postgresql-42.5.0.jar
jardir = /opt/spark_hadoop_3/jars/postgresql-42.6.0.jar
master = local[1]
partitions = 4
format = jdbc
savemode = Append
[AIRFLOW]
#--set the AIRFLOW_HOME directory $path to save the dags
# default: airflow_home = ~/airflow
airflowdirhome = ~/airflow
#--set the aiflow username and password
# default: username=rezaware password=rezaware
airflowappadmin = rezaware
airflowapppswd = rezaware
#--set the airflow email to communicate logs and errors to the admin
# default: airflow_email = [email protected]
airflowemail = [email protected]
#--set the airflow database as postgres and mage sure the change
# aiflow.cfg parameters
# sql_alchemy_conn =postgresql+psycopg2://airflow@localhost:5432/airflow
# executor = LocalExecutor
airflowdbuser = rezawareflow
airflowdbpswd = rezawareflow
[AWSAUTH]
#--AWS security key and credentials
credprofile = hero
credfilepath = /home/nuwan/.aws/credentials
region = ap-southeast-1
iampolicy =
iamuser =
[GOOGLE]
#--Goole Cloud requires a project to be specified
# in the credentials
projectid=glass-watch-369111
[DATABASE]
#--database types: mysql, postgresql (default: postgres)
dbtype = postgresql
#--hostIP default 127.0.0.1
dbhostip = 127.0.0.1
#--port default 5432
dbport = 5432
#--database driver
# postgresql for apache spark: 'org.postgresql.Driver'
dbdriver = org.postgresql.Driver
#--database name
dbname = tip
#--schema name
dbschema = warehouse
#--username and password to connect
# default db_user=postgres, db_pswd = postgres
dbuser =
dbpswd =
[NOSQLDB]
##--Note dbuser, dbpswd, dbauthsource should be created
dbtype = mongodb
dbHostIP = 127.0.0.1
dbport = 27017
##--necessary for the spark connector
dbformat = mongo
##--set a default database
dbname = admin
##--authentiction requires username, password,
## authentication database, and mechanisem
dbuser =
dbpswd =
dbauthsource = admin
dbauthmechanism = SCRAM-SHA-256
##--setting up a keyfile for authentication
dbtls = false
dbtlsCertKeyFile =
dbtlsCAFile =
[TIMEZONE]
default = utc-8:00
[CURRENCY]
default = USD
currabrv = US$
[LOGGER]
#--directory path to store logs
# defaulat logs/
path = logs/
file = app.log
#--logging level to set DEBUG, ERROR, WARNING, INFO
level = DEBUG
#--append or write over the logs
# default: 'a' to append, 'w+' to overwrite
mode = w+
#--loggin format
# default: %(asctime)s - %(name)s - %(levelname)s - %(message)s'
# logFormat = %(asctime)s - %(name)s - %(levelname)s - %(message)s
format = asctime,name,levelname,message