migrated to new yaml config

Author: johnbuluba

.gitignore

@@ -59,6 +59,7 @@ target/
 
 .idea/
 config.json
+config.yaml
 ignorelist
 log
 

Pipfile

@@ -9,6 +9,7 @@ requests = ">=2.7.0"
 requests-oauthlib = ">=0.5.0"
 python-dateutil = ">=2.4.2"
 "pushbullet.py" = ">=0.8.1"
+confuse = "*"
 
 [dev-packages]
 requests-mock = "*"

Pipfile.lock

@@ -1,10 +1,12 @@
 certifi==2018.1.18
 chardet==3.0.4
+confuse==0.4.0
 idna==2.6
 oauthlib==2.0.7
 pushbullet.py==0.11.0
 python-dateutil==2.7.2
 python-magic==0.4.15
+pyyaml==3.12
 requests-oauthlib==0.8.0
 requests==2.18.4
 six==1.11.0

requirements.txt

@@ -8,7 +8,7 @@
 logging.disable(logging.ERROR)
 
 
-class TestConfig(unittest.TestCase):
+class TestConfigJson(unittest.TestCase):
     config_data = '''{"search_queries":["test"],"follow_keywords":["test"],"fav_keywords":["test"],"consumer_key":"test","consumer_secret":"test","access_token_key":"test","access_token_secret":"test","retweet_interval":1,"retweet_random_margin":1,"scan_interval":1,"clear_queue_interval":1,"max_queue":1,"rate_limit_update_interval":1,"min_ratelimit_percent":1, "min_quote_similarity":0.80, "max_quote_depth": 5,"blocked_users_update_interval":1,"max_follows":1, "check_mentions_interval":1, "pushbullet_token": "test"}'''
 
     def test_load(self):
@@ -41,3 +41,11 @@ def test_save_tokens(self):
             handle = m()
             handle.write.assert_any_call('"new_token"')
             handle.write.assert_any_call('"new_secret"')
+
+
+class TestConfig(unittest.TestCase):
+
+    def test_simple_config(self):
+        Config()
+
+

tests/test_config.py

@@ -25,7 +25,7 @@ class Follow(ActionABC):
 
     def process(self, post):
         text = post['full_text']
-        keywords = sum((create_keyword_mutations(x) for x in Config.follow_keywords), [])
+        keywords = sum((create_keyword_mutations(x) for x in Config.get_config().actions.follow.keywords), [])
         if any(x in text.lower() for x in keywords):
             self.remove_oldest_follow()
             self.client.follow(post['user']['screen_name'])
@@ -38,7 +38,7 @@ def remove_oldest_follow(self):
 
         follows = self.client.get_friends_ids()
 
-        if len(follows) > Config.max_follows:
+        if len(follows) > Config.get_config().actions.follow.max_following:
             r = self.client.unfollow(follows[-1])
             logger.info('Unfollowed: {0}'.format(r['screen_name']))
 
@@ -50,7 +50,7 @@ class Favorite(ActionABC):
 
     def process(self, post):
         text = post['full_text']
-        keywords = sum((create_keyword_mutations(x) for x in Config.fav_keywords), [])
+        keywords = sum((create_keyword_mutations(x) for x in Config.get_config().actions.favorite.keywords), [])
         if any(x in text.lower() for x in keywords):
             r = self.client.favorite(post['id'])
             logger.info("Favorite: {0}".format(post['id']))

yatcobot/actions.py

@@ -1,5 +1,6 @@
 import difflib
 import logging
+from collections import OrderedDict
 
 from .actions import Favorite, Follow
 from .client import TwitterClient, TwitterClientRetweetedException
@@ -19,9 +20,10 @@ def __init__(self, ignore_list_file):
 
         self.ignore_list = IgnoreList(ignore_list_file)
         self.post_queue = PostQueue()
-        self.client = TwitterClient(Config.consumer_key, Config.consumer_secret,
-                                    Config.access_token_key,
-                                    Config.access_token_secret)
+        self.client = TwitterClient(Config.get_config()['consumer_key'],
+                                    Config.get_config()['consumer_secret'],
+                                    Config.get_config()['access_token_key'],
+                                    Config.get_config()['access_token_secret'])
         self.scheduler = PeriodicScheduler()
         self.notification = NotificationService()
         self.actions = [Follow(self.client), Favorite(self.client)]
@@ -60,7 +62,7 @@ def enter_contest(self):
     def clear_queue(self):
         """Removes the extraneous posts from the post_queue"""
 
-        to_delete = len(self.post_queue) - Config.max_queue
+        to_delete = len(self.post_queue) - Config.get_config().search.max_queue
 
         if to_delete > 0:
             for i in range(to_delete):
@@ -82,9 +84,16 @@ def scan_new_contests(self):
 
         logger.info("=== SCANNING FOR NEW CONTESTS ===")
 
-        for search_query in Config.search_queries:
+        for search_query in Config.get_config().search.queries:
+
+            if isinstance(search_query, str):
+                results = self.client.search_tweets(search_query, 50)
+            elif isinstance(search_query, OrderedDict):
+                search_query, settings = search_query.popitem()
+                results = self.client.search_tweets(search_query, 50, language=settings['lang'])
+            else:
+                raise ValueError("Uknown type of query {}".format(str(search_query)))
 
-            results = self.client.search_tweets(search_query, 50, language=Config.search_language)
             logger.info("Got {} new results for: {}".format(len(results), search_query))
 
             for post in results:
@@ -124,12 +133,13 @@ def check_new_mentions(self):
     def run(self):
         """Run the bot as a daemon. This is blocking command"""
 
-        self.scheduler.enter(Config.clear_queue_interval, 1, self.clear_queue)
-        self.scheduler.enter(Config.rate_limit_update_interval, 2, self.client.update_ratelimits)
-        self.scheduler.enter(Config.blocked_users_update_interval, 3, self.update_blocked_users)
-        self.scheduler.enter(Config.check_mentions_interval, 4, self.check_new_mentions)
-        self.scheduler.enter(Config.scan_interval, 5, self.scan_new_contests)
-        self.scheduler.enter_random(Config.retweet_interval, Config.retweet_random_margin, 6, self.enter_contest)
+        self.scheduler.enter(Config.get_config().scheduler.clear_queue_interval, 1, self.clear_queue)
+        self.scheduler.enter(Config.get_config().scheduler.rate_limit_update_interval, 2, self.client.update_ratelimits)
+        self.scheduler.enter(Config.get_config().scheduler.blocked_users_update_interval, 3, self.update_blocked_users)
+        self.scheduler.enter(Config.get_config().scheduler.check_mentions_interval, 4, self.check_new_mentions)
+        self.scheduler.enter(Config.get_config().scheduler.search_interval, 5, self.scan_new_contests)
+        self.scheduler.enter_random(Config.get_config().scheduler.retweet_interval,
+                                    Config.get_config().scheduler.retweet_random_margin, 6, self.enter_contest)
 
         # Init the program
         self.scheduler.run()
@@ -153,15 +163,15 @@ def _get_quoted_tweet(self, post):
         :param post: The post to check if its a quote
         :return: If it isnt a quote the argument, otherwise the original tweet
         """
-        for i in range(Config.max_quote_depth):
+        for i in range(Config.get_config().search.max_quote_depth):
             # If it hasnt quote return the post
             if 'quoted_status' not in post:
                 return post
 
             quote = post['quoted_status']
             diff = difflib.SequenceMatcher(None, post['full_text'], quote['full_text']).ratio()
             # If the texts are similar continue
-            if diff >= Config.min_quote_similarity:
+            if diff >= Config.get_config().search.min_quote_similarity:
                 logger.debug('{} is a quote, following to next post. Depth from original post {}'.format(post['id'], i))
                 quote = self.client.get_tweet(quote['id'])
                 # If its a quote of quote, get next quote and continue

yatcobot/bot.py

@@ -40,6 +40,5 @@ def main():
         create_logger(logging.INFO, args.logfile)
 
     Config.load(args.config)
-
     bot = Yatcobot(args.ignore_list)
     bot.run()

yatcobot/cli.py

@@ -62,7 +62,7 @@ def check_limit(self, endpoint):
         endpoint_limits = self[self._get_internal_endpoint_name(endpoint)]
 
         # if over threshold sleep untill reset
-        if endpoint_limits['percent'] < Config.min_ratelimit_percent:
+        if endpoint_limits['percent'] < Config.get_config().min_ratelimit_percent:
 
             reset_time = endpoint_limits['reset']
             now = int(datetime.datetime.now().strftime('%s'))

yatcobot/client.py

@@ -1,7 +1,78 @@
 import json
+import confuse
 
 
-class Config:
+class Config(confuse.AttrDict):
+
+    template = {
+        'consumer_key': confuse.String(),
+        'consumer_secret': confuse.String(),
+        'access_token_key': confuse.String(),
+        'access_token_secret': confuse.String(),
+        'min_ratelimit_percent': confuse.Integer(),
+
+        'search': {
+            'queries': confuse.TypeTemplate(list),
+            'priority_keywords': confuse.StrSeq(),
+            'max_queue': confuse.Integer(),
+            'max_quote_depth': confuse.Integer(),
+            'min_quote_similarity': confuse.Number(),
+        },
+
+        'actions': {
+            'follow': {
+                'enabled': confuse.TypeTemplate(bool),
+                'keywords': confuse.StrSeq(),
+                'max_following': confuse.Integer(),
+            },
+            'favorite': {
+                'enabled': confuse.TypeTemplate(bool),
+                'keywords': confuse.StrSeq()
+            },
+        },
+
+        'scheduler': {
+            'search_interval': confuse.Integer(),
+            'retweet_interval': confuse.Integer(),
+            'retweet_random_margin': confuse.Integer(),
+            'blocked_users_update_interval': confuse.Integer(),
+            'clear_queue_interval': confuse.Integer(),
+            'rate_limit_update_interval': confuse.Integer(),
+            'check_mentions_interval': confuse.Integer(),
+        },
+
+        'notifiers': {
+            'pushbullet': {
+                'enabled': confuse.TypeTemplate(bool),
+                'token': confuse.String()
+            }
+        }
+    }
+
+    _valid = None
+
+    @staticmethod
+    def get_config():
+        """
+        Gets the static config object
+        :return:
+        """
+        if Config._valid is None:
+            raise ValueError("Configuration not loaded")
+        return Config._valid
+
+    @staticmethod
+    def load(filename):
+        """
+        Loads a file and imports the settings
+        :param filename: the file to import
+        """
+        config = confuse.LazyConfig('Yatcobot', __name__)
+        config.set_file(filename)
+        Config._valid = config.get(Config.template)
+
+
+class ConfigJson:
     """Class that contains all  config variables. It loads user values from a json file """
 
     # Default values
@@ -42,8 +113,6 @@ def load(filename):
             if value == "":
                 value = None
             setattr(Config, key, value)
-        b = Config.search_language
-        a = 5
 
     @staticmethod
     def save_user_tokens(filename, token, secret):

yatcobot/config.py

@@ -0,0 +1,42 @@
+min_ratelimit_percent: 10
+
+consumer_key: null
+consumer_secret: null
+access_token_key: null
+access_token_secret: null
+
+search:
+  queries:
+    - RT to win
+    - Retweet and win
+    - Διαγωνισμός:
+        lang: el
+  max_queue: 100
+  max_quote_depth: 20
+  min_quote_similarity: 0.5
+  priority_keywords: ["ps4", "pc"]
+
+actions:
+  follow:
+    enabled: true
+    keywords: ["follow", "follower"]
+    max_following: 1950
+  favorite:
+    enabled: true
+    keywords: ["fav", "favorite"]
+
+scheduler:
+  search_interval: 5400
+  retweet_interval: 600
+  retweet_random_margin: 60
+  blocked_users_update_interval: 300
+  clear_queue_interval: 60
+  rate_limit_update_interval: 60
+  check_mentions_interval: 600
+
+notifiers:
+  pushbullet:
+    enabled: false
+    token: test
+
+

yatcobot/config_default.yaml

@@ -72,13 +72,13 @@ def notify(self, title, message):
 
         @staticmethod
         def is_enabled():
-            if Config.pushbullet_token:
+            if Config.get_config().notifiers.pushbullet.enabled:
                 return True
             return False
 
         @classmethod
         def from_config(cls):
-            return cls(Config.pushbullet_token)
+            return cls(Config.get_config().notifiers.pushbullet.enabled)
 
 except ImportError:
     logger.warning("Could not import pushbullet.py. Pushbullet notification is disabled")

yatcobot/notifier.py

@@ -61,7 +61,7 @@ def get_keywords_score(self):
             score = 0
             text = post['full_text'].lower()
 
-            for keyword in Config.priority_keywords:
+            for keyword in Config.get_config().search.priority_keywords:
                 keyword = keyword.lower()
                 score += text.count(keyword)