🏡 index : ~doyle/dave.git

0ade70523a6348f4af23b6df51b1dba083e6bec6 [patch]

fd0fe4556214579ddfb5684f3c9dfeca37d342f8

fe84772ca886c4252fb29e1991f7a601f545d1d9

0ade70523a6348f4af23b6df51b1dba083e6bec6.tar.gz

Remove markov & socks proxying

Diff

 alembic.ini                                            |  3 +--
 requirements.txt                                       |  4 ++--
 dave/config.py                                         | 33 +--------------------------------
 dave/dave.py                                           | 16 ----------------
 dave/models.py                                         | 10 ----------
 dave/module.py                                         |  2 --
 dave/modules/btc.py                                    |  3 +--
 dave/modules/markov.py                                 | 38 --------------------------------------
 dave/modules/nice.py                                   |  2 +-
 dave/modules/pollen.py                                 |  2 --
 dave/modules/sed.py                                    | 10 ++++++----
 dave/modules/title.py                                  |  2 --
 dave/modules/urbandictionary.py                        |  2 --
 dave/modules/weather.py                                |  6 ------
 dave/modules/wolfram.py                                |  2 --
 migrations/versions/7836665787cd_add_messages_table.py | 38 --------------------------------------
 16 files changed, 12 insertions(+), 161 deletions(-)

diff --git a/alembic.ini b/alembic.ini
index 92affd9..1efc2f6 100644
--- a/alembic.ini
+++ a/alembic.ini
@@ -29,8 +29,7 @@
# are written from script.py.mako

# output_encoding = utf-8


sqlalchemy.url = postgresql://homestead:secret@192.168.10.10/dave

sqlalchemy.url = postgresql://dave:dave@127.0.0.1/dave

# Logging configuration

[loggers]
diff --git a/requirements.txt b/requirements.txt
index f3351df..5df71a0 100644
--- a/requirements.txt
+++ a/requirements.txt
@@ -1,5 +1,5 @@
twisted==16.2.0
pyopenssl==16.0.0
twisted==17.5.0
pyopenssl==16.2.0
service_identity==16.0.0
enum34==1.1.6
beautifulsoup4==4.4.1
diff --git a/dave/config.py b/dave/config.py
index ae1fba3..b1e7a6c 100644
--- a/dave/config.py
+++ a/dave/config.py
@@ -1,13 +1,11 @@
import redis
import json
import os.path
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import socket
import socks
import markovify
import nltk
from models import Message
import re

# read the config file
@@ -20,36 +18,7 @@
redis = redis.StrictRedis(host=config["redis"]["host"], port=config["redis"]["port"],
                          db=config["redis"]["db"])

# set everything to use our socks proxy
proxy = config['socks'].split(":")

default_socket = socket.socket

if proxy[0]:
    socks.set_default_proxy(socks.SOCKS5, proxy[0], int(proxy[1]))
    proxied_socket = socks.socksocket
else:
    proxied_socket = socket.socket

# connect to the database
db = create_engine(config["database"])
Session = sessionmaker(bind=db)
session = Session()

# setup the markov chain
class PText(markovify.NewlineText):
    def word_split(self, sentence):
        words = re.split(self.word_split_pattern, sentence)
        words = [ "::".join(tag) for tag in nltk.pos_tag(words) ]
        return words

    def word_join(self, words):
        sentence = " ".join(word.split("::")[0] for word in words)
        return sentence

if not redis.exists("markov:chain"):
    markov = PText("\n".join([m[0] for m in session.query(Message.message).all()]), state_size=2)
    redis.setex("markov:chain", 500, markov.chain.to_json())
else:
    markov = PText("\n".join([m[0] for m in session.query(Message.message).all()]),
                   state_size=2, chain=markovify.Chain.from_json(redis.get("markov:chain")))
session = Session()
diff --git a/dave/dave.py b/dave/dave.py
index 777dcc0..e7b9099 100644
--- a/dave/dave.py
+++ a/dave/dave.py
@@ -9,7 +9,6 @@
import re
import config
import datetime
from models import Message
from twisted.internet.threads import deferToThread


@@ -73,21 +72,6 @@
                                    run.append((val, match.groups()))
                                else:
                                    method = (priority, val, match.groups())

        if not msg.startswith(self.nickname) and channel.startswith("#"):
            try:
                # not a direct command so we'll save the response to the db
                db_msg = Message(
                    nick=nick,
                    channel=channel,
                    message=msg.decode('utf-8', 'ignore').encode('utf-8'),
                    userhost=user.split("!", 1)[1],
                    created_at=datetime.datetime.now()
                )
                config.session.add(db_msg)
                config.session.commit()
            except:
                log.err()

        if method[1] is not None:
            # we matched a command
diff --git a/dave/models.py b/dave/models.py
index 2ff48ff..205de4f 100644
--- a/dave/models.py
+++ a/dave/models.py
@@ -9,13 +9,3 @@
    id = Column(Integer, primary_key=True)
    nick = Column(String)
    location = Column(String)

class Message(Base):
    __tablename__ = "messages"

    id = Column(Integer, primary_key=True)
    nick = Column(String)
    channel = Column(String)
    message = Column(String)
    userhost = Column(String)
    created_at = Column(DateTime)
diff --git a/dave/module.py b/dave/module.py
index b753403..6473bc8 100644
--- a/dave/module.py
+++ a/dave/module.py
@@ -88,9 +88,7 @@
    """Decorate a function to always proxy data flowing from it."""
    def proxifier(function):
        def wrapper(*args, **kw):
            socket.socket = config.proxied_socket
            output = function(*args, **kw)
            socket.socket = config.default_socket
            return output

        return wrapper
diff --git a/dave/modules/btc.py b/dave/modules/btc.py
index 5da2825..6697e60 100644
--- a/dave/modules/btc.py
+++ a/dave/modules/btc.py
@@ -48,8 +48,7 @@
            market,
            babel.numbers.format_currency(decimal.Decimal(data['last']) * multiplier,
                                          currency)
        ).encode('utf-8')
        for market, data in p['btc'][currencyKey].iteritems()
        ).encode('utf-8') for market, data in p['btc'][currencyKey].iteritems()
    ])

    prices += u'. average: {}'.format(
diff --git a/dave/modules/markov.py b/dave/modules/markov.py
deleted file mode 100644
index 0773db3..0000000 100644
--- a/dave/modules/markov.py
+++ /dev/null
@@ -1,38 +1,0 @@
# -*- coding: utf-8 -*-
"""Chatter bot using Markov chains."""
import dave.module
import dave.config
import random
import re
from dave.models import Message

FACTOR = 0.05


@dave.module.help("Syntax: babble [optional seeds].")
@dave.module.command(["babble"], "?(.*)")
@dave.module.priority(dave.module.Priority.HIGHEST)
def babble(bot, args, sender, source):
    msg = args[0].strip()

    if msg:
        args = msg.split(" ")

        if len(args) == 2:
            # require two seeds
            resp = dave.config.markov.make_sentence(tuple(dave.config.markov.word_split(" ".join(args))), tries=10000)
        else:
            resp = None
    else:
        resp = dave.config.markov.make_sentence(tries=10000)

    if not resp is None:
        bot.msg(source, resp.encode("utf-8"))


#@dave.module.match(r"(.*)")
#@dave.module.always_run()
#def random_babble(bot, args, sender, source):
#    if not args[0].startswith(bot.nickname) and random.random() < FACTOR:
#        # every (factor*100)% of messages should be babbled about
#        babble(bot, args, sender, source)
diff --git a/dave/modules/nice.py b/dave/modules/nice.py
index 3e9248c..ce4f967 100644
--- a/dave/modules/nice.py
+++ a/dave/modules/nice.py
@@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
"""Nice."""
import dave.module


@dave.module.help("Nice.", name="donk")
@dave.module.help("Nice.", name="nice")
@dave.module.match(r"^nice\.?$")
@dave.module.priority(dave.module.Priority.LOW)
def donk(bot, args, sender, source):
diff --git a/dave/modules/pollen.py b/dave/modules/pollen.py
index 49ea6d1..6c20de2 100644
--- a/dave/modules/pollen.py
+++ a/dave/modules/pollen.py
@@ -19,10 +19,8 @@
    text = None

    if not dave.config.redis.exists("pollen:{}".format(postcode)):
        socket.socket = dave.config.proxied_socket
        res = br.open("https://www.bbc.co.uk/weather/{}".format(postcode))
        data = res.get_data()
        socket.socket = dave.config.default_socket

        soup = BeautifulSoup(data, "html.parser")
        element = soup.find_all("div", class_="environmental-index pollen-index")
diff --git a/dave/modules/sed.py b/dave/modules/sed.py
index 3307bff..13ae71f 100644
--- a/dave/modules/sed.py
+++ a/dave/modules/sed.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
"""Pass any messages beginning with 'sed' to GNU sed."""
import subprocess
import dave.module
import dave.config
from twisted.words.protocols.irc import assembleFormattedText, attributes as A
import re


@@ -20,9 +20,11 @@
            f = f | re.IGNORECASE

        try:
            replace = re.sub(args[3], args[5], msg, count=0 if 'g' in flags else 1,
            replace = re.sub(args[3],
                             assembleFormattedText(A.bold[args[5]]),
                             msg,
                             count=0 if 'g' in flags else 1,
                             flags=f)
            print(replace)
        except Exception as e:
            bot.reply(source, sender,
                      "There was a problem with your sed command: {}".format(str(e)))
@@ -30,7 +32,7 @@

        if replace.strip() != msg:
            msg = replace.strip()
            bot.msg(source, "{} meant: {}".format(sender, msg))
            bot.msg(source, "<{}> {}".format(sender, msg))
            dave.config.redis.lset(key, i, msg)
            return

diff --git a/dave/modules/title.py b/dave/modules/title.py
index ba9ad18..6e54c4c 100644
--- a/dave/modules/title.py
+++ a/dave/modules/title.py
@@ -23,10 +23,8 @@

    for match in matches:
        if not dave.config.redis.exists("site:{}".format(match)):
            socket.socket = dave.config.proxied_socket
            res = br.open(match)
            data = res.get_data()
            socket.socket = dave.config.default_socket

            soup = BeautifulSoup(data, "html.parser")
            title = soup.title
diff --git a/dave/modules/urbandictionary.py b/dave/modules/urbandictionary.py
index 1c7e8ab..6a956f6 100644
--- a/dave/modules/urbandictionary.py
+++ a/dave/modules/urbandictionary.py
@@ -24,7 +24,6 @@
        return

    if not dave.config.redis.exists("urban_query:{}".format(query)):
        socket.socket = dave.config.proxied_socket
        url = "https://mashape-community-urban-dictionary.p.mashape.com/define?term={}".format(quote_plus(query))
        r = requests.get(url, headers={
            "X-Mashape-Key": dave.config.config["api_keys"]["mashape"],
@@ -32,7 +31,6 @@
        })

        resp = r.json()
        socket.socket = dave.config.default_socket
        dave.config.redis.setex("urban_query:{}".format(query), 86400, pickle.dumps(resp))
    else:
        resp = pickle.loads(dave.config.redis.get("urban_query:{}".format(query)))
diff --git a/dave/modules/weather.py b/dave/modules/weather.py
index 8d11237..52868d8 100644
--- a/dave/modules/weather.py
+++ a/dave/modules/weather.py
@@ -65,14 +65,12 @@
    key = "location:{}".format(location.lower())

    if not dave.config.redis.exists(key):
        socket.socket = dave.config.proxied_socket
        r = requests.get("https://maps.googleapis.com/maps/api/geocode/json?address={}&key={}".format(
            quote_plus(location),
            dave.config.config["api_keys"]["google_maps"]
        ))

        geocode = r.json()
        socket.socket = dave.config.default_socket

        dave.config.redis.set(key, pickle.dumps(geocode))
    else:
@@ -112,7 +110,6 @@
                                  geocode["results"][0]["geometry"]["location"]["lng"])

    if not dave.config.redis.exists(key):
        socket.socket = dave.config.proxied_socket
        r = requests.get("http://api.timezonedb.com/?key={}&lat={}&lng={}&format=json".format(
            dave.config.config["api_keys"]["timezonedb"],
            geocode["results"][0]["geometry"]["location"]["lat"],
@@ -120,7 +117,6 @@
        ))

        timezone = r.json()
        socket.socket = dave.config.default_socket

        dave.config.redis.set(key, pickle.dumps(timezone))
    else:
@@ -135,7 +131,6 @@
                                 geocode["results"][0]["geometry"]["location"]["lng"])

    if not dave.config.redis.exists(key):
        socket.socket = dave.config.proxied_socket
        r = requests.get("https://api.forecast.io/forecast/{}/{},{}?units=uk2".format(
            dave.config.config["api_keys"]["forecast.io"],
            geocode["results"][0]["geometry"]["location"]["lat"],
@@ -143,7 +138,6 @@
        ))

        json = r.json()
        socket.socket = dave.config.default_socket

        dave.config.redis.setex(key, 300, pickle.dumps(json))
    else:
diff --git a/dave/modules/wolfram.py b/dave/modules/wolfram.py
index 781bf41..b219820 100644
--- a/dave/modules/wolfram.py
+++ a/dave/modules/wolfram.py
@@ -21,10 +21,8 @@
        return

    if query:
        socket.socket = dave.config.proxied_socket
        client = wolframalpha.Client(dave.config.config["api_keys"]["wolfram"])
        res = client.query(query)
        socket.socket = dave.config.default_socket

        if len(res.pods) > 0:
            resultpod = next(res.results)
diff --git a/migrations/versions/7836665787cd_add_messages_table.py b/migrations/versions/7836665787cd_add_messages_table.py
deleted file mode 100644
index c220fbb..0000000 100644
--- a/migrations/versions/7836665787cd_add_messages_table.py
+++ /dev/null
@@ -1,38 +1,0 @@
"""Add messages table

Revision ID: 7836665787cd
Revises: 0689a529bce0
Create Date: 2016-11-12 08:20:05.648069

"""

# revision identifiers, used by Alembic.
revision = '7836665787cd'
down_revision = '0689a529bce0'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa


def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('messages',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('nick', sa.String(), nullable=True),
    sa.Column('channel', sa.String(), nullable=True),
    sa.Column('message', sa.String(), nullable=True),
    sa.Column('userhost', sa.String(), nullable=True),
    sa.Column('created_at', sa.DateTime(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.drop_constraint(u'locations_nick_key', 'locations', type_='unique')
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_unique_constraint(u'locations_nick_key', 'locations', ['nick'])
    op.drop_table('messages')
    ### end Alembic commands ###