суббота, 18 марта 2017 г.

Remote debugging in Python

Allows debugging of child subprocesses, author of script is Bertrand Janin. (Yes, I know that some IDE has own way: PyCharm for example, but this is IDE-independ way).

Next script must be put into Libs of Python install dir, then in app type somewhere:

import rpdb
rpdb.set_trace()

And run telnet to 127.0.0.1:4444 and get (pdb) console in telnet window.

"""Remote Python Debugger (pdb wrapper)."""

__author__ = "Bertrand Janin <b@janin.com>"
__version__ = "0.1.6"

import pdb
import socket
import threading
import signal
import sys
import traceback
from functools import partial

DEFAULT_ADDR = "127.0.0.1"
DEFAULT_PORT = 4444


class FileObjectWrapper(object):
    def __init__(self, fileobject, stdio):
        self._obj = fileobject
        self._io = stdio

    def __getattr__(self, attr):
        if hasattr(self._obj, attr):
            attr = getattr(self._obj, attr)
        elif hasattr(self._io, attr):
            attr = getattr(self._io, attr)
        else:
            raise AttributeError("Attribute %s is not found" % attr)
        return attr


class Rpdb(pdb.Pdb):

    def __init__(self, addr=DEFAULT_ADDR, port=DEFAULT_PORT):
        """Initialize the socket and initialize pdb."""

        # Backup stdin and stdout before replacing them by the socket handle
        self.old_stdout = sys.stdout
        self.old_stdin = sys.stdin
        self.port = port

        # Open a 'reusable' socket to let the webapp reload on the same port
        self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
        self.skt.bind((addr, port))
        self.skt.listen(1)

        # Writes to stdout are forbidden in mod_wsgi environments
        try:
            sys.stderr.write("pdb is running on %s:%d\n"
                             % self.skt.getsockname())
        except IOError:
            pass

        (clientsocket, address) = self.skt.accept()
        handle = clientsocket.makefile('rw')
        pdb.Pdb.__init__(self, completekey='tab',
                         stdin=FileObjectWrapper(handle, self.old_stdin),
                         stdout=FileObjectWrapper(handle, self.old_stdin))
        sys.stdout = sys.stdin = handle
        OCCUPIED.claim(port, sys.stdout)

    def shutdown(self):
        """Revert stdin and stdout, close the socket."""
        sys.stdout = self.old_stdout
        sys.stdin = self.old_stdin
        OCCUPIED.unclaim(self.port)
        self.skt.close()

    def do_continue(self, arg):
        """Clean-up and do underlying continue."""
        try:
            return pdb.Pdb.do_continue(self, arg)
        finally:
            self.shutdown()

    do_c = do_cont = do_continue

    def do_quit(self, arg):
        """Clean-up and do underlying quit."""
        try:
            return pdb.Pdb.do_quit(self, arg)
        finally:
            self.shutdown()

    do_q = do_exit = do_quit

    def do_EOF(self, arg):
        """Clean-up and do underlying EOF."""
        try:
            return pdb.Pdb.do_EOF(self, arg)
        finally:
            self.shutdown()


def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT, frame=None):
    """Wrapper function to keep the same import x; x.set_trace() interface.

    We catch all the possible exceptions from pdb and cleanup.

    """
    try:
        debugger = Rpdb(addr=addr, port=port)
    except socket.error:
        if OCCUPIED.is_claimed(port, sys.stdout):
            # rpdb is already on this port - good enough, let it go on:
            sys.stdout.write("(Recurrent rpdb invocation ignored)\n")
            return
        else:
            # Port occupied by something else.
            raise
    try:
        debugger.set_trace(frame or sys._getframe().f_back)
    except Exception:
        traceback.print_exc()


def _trap_handler(addr, port, signum, frame):
    set_trace(addr, port, frame=frame)


def handle_trap(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
    """Register rpdb as the SIGTRAP signal handler"""
    signal.signal(signal.SIGTRAP, partial(_trap_handler, addr, port))


def post_mortem(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
    debugger = Rpdb(addr=addr, port=port)
    type, value, tb = sys.exc_info()
    traceback.print_exc()
    debugger.reset()
    debugger.interaction(None, tb)


class OccupiedPorts(object):
    """Maintain rpdb port versus stdin/out file handles.

    Provides the means to determine whether or not a collision binding to a
    particular port is with an already operating rpdb session.

    Determination is according to whether a file handle is equal to what is
    registered against the specified port.
    """

    def __init__(self):
        self.lock = threading.RLock()
        self.claims = {}

    def claim(self, port, handle):
        self.lock.acquire(True)
        self.claims[port] = id(handle)
        self.lock.release()

    def is_claimed(self, port, handle):
        self.lock.acquire(True)
        got = (self.claims.get(port) == id(handle))
        self.lock.release()
        return got

    def unclaim(self, port):
        self.lock.acquire(True)
        del self.claims[port]
        self.lock.release()

# {port: sys.stdout} pairs to track recursive rpdb invocation on same port.
# This scheme doesn't interfere with recursive invocations on separate ports -
# useful, eg, for concurrently debugging separate threads.
OCCUPIED = OccupiedPorts()

Small Python tricks

Usefull modules and tools

  • pycallgraph
  • byteplay
  • objgraph
  • twisted.manhole
  • rfoo.utils.rconsole
  • pydevd
  • heapy (2.7)
  • tracemalloc
  • sfood (+modviz), sfood-imports, etc...
  • snakefood (http://furius.ca/snakefood/) - dependencies
  • rpdb (remote debuging)

PyOpenSSL on CentOS

sudo yum install openssl-devel
sudo yum install libffi-devel
pip install pyopenssl

Log all imports

Run python PYTHONVERBOSE=5 python ...

Documentation per versions

https://www.python.org/doc/versions/

Small shell tricks

Directories tree view

Set it to your .bashrc and run as tree [DIR].

tree() {
find "${1:-.}" -type d -print 2>/dev/null|awk '!/\.$/ {for (i=1;i<NF;i++){d=length($i);if ( d < 5  && i != 1 )d=5;printf("%"d"s","|")}print "---"$NF}'  FS='/'
}

Patch selected file only

If you have big patch file with many files there, to patch only one (filename.py in example):

filterdiff -i '*filename.py' patch.diff | patch -pN

and to list all files in patch file use lsdiff or filterdiff --list.

Test HEAD HTTP with curl

curl -i -X HEAD http://...

Paging in FAR

view:<command
edit:<command

and to enable scrolling start with far -w.

Run alternate shell commands

This script run one of available command:

#!/bin/sh

trycmd() {
    a0=$1; shift
    for c in $*; do $c $a0 2>/dev/null && break; done
}

# usage: run one of: pipx|pip0|pip1|pip3|pip with arg: list
trycmd list pipx pip0 pip1 pip3 pip

Frozen keyboard in PyCharm (Linux)

Bug in forever buggy Java:

ibus restart

Remove passphrase from SSH key

openssl rsa -in ~/.ssh/id_rsa -out ~/.ssh/id_rsa_new
cp ~/.ssh/id_rsa ~/.ssh/id_rsa.backup
rm ~/.ssh/id_rsa
cp ~/.ssh/id_rsa_new ~/.ssh/id_rsa
chmod 400 ~/.ssh/id_rsa

Problem with RPC/NFS mount after update CentOS 7

After some digging I finally found that the reason for this is a buggy systemd unit file in RHEL/CentOS 7.

This is a bug in nfs-server.service and he fix is to copy nfs-server.service to /etc/systemd/system and replace all occurrences of "rpcbind.target" with "rpcbind.service".

See this bug for reference: https://bugzilla.redhat.com/show_bug.cgi?id=1171603

To allow connecting to CentOS

If there is trouble try to stop iptables:

service iptables start iptables flush service iptables stop

Script to run jobs suite remotely via screen

Often in needed to run some jobs remotely, for example, tests on remote test environment. I used different custom "daemon"-like scripts for this, monitoring the jobs/tests, but another and simple solution for Linux is to use screen tool. It supports sessions, detaching, so is good for such kind of tasks. Here is the script which runs multiple jobs on remote host (setted via $RMHOST env. var). Its syntax is:

  -cls
  -run JOB-NUMBER 'JOB-COMMAND'
  -runall < FILE-WITH-COMMANDS
  -submit < FILE-WITH-COMMANDS
Used env. vars: \$USER (login), \$RMHOST (hostname)

so to run it you must:

  • set $RMHOST env. var
  • create file with commands which will be ran remotely: each command is on own line and can have any $ variables for substitution: they will be substituted locally
  • then call script as V1=a V2=b script -submit < created_file. This will substitute V1 and V2 in created_file and send resulting file over scp to remote host $RMHOST (you must have configured public-key access to host), then will send itself and call itself remotely with command -runall.

After it you can go to host, call screen -r to see your created session, to switch between windows (each job is ran on separate window). You will have local $LOG file with decscription of running commands and remote log file per job with info about what and when was run (job-<DATE>-<JOB_NUMBER>.log). Yes, each job has number/index related to line number of job suite file. If you want you can include generating of screen log file too (with -L option IMO).

While you are on remote host, you can run this script (which was scp-ed) with -cls command: it will remove all log files there.

That's it.

#!/bin/sh
SCR=`basename "$0"`
SESSION=${USER}_screen
LOG=rmjobs.log

_wrapjob() {
    _wrapjob_f=job-`date +%s`-$1.log
    echo "Started $1 at `date`: $2" > $_wrapjob_f
    echo '===============================================================================' >> $_wrapjob_f
    echo >> $_wrapjob_f
    $2|tee -a $_wrapjob_f
}

_runjob() {
    [ $1 = "0" ] && {
        # add -L to both if screelog.* logs are needed
        screen -dmS $SESSION -t "job$1" sh -c "~/$SCR -run $1 \"$2\";sh"
    } || {
        screen -S $SESSION -p 0 -X screen -t "job$1" sh -c "~/$SCR -run $1 \"$2\";sh"
    }
}

case "$1" in
    -cls) rm -f screenlog.*; rm -f job*.log;;
    -run) _wrapjob "$2" "$3";;
    -runall)
        i=0
        rm -f $LOG 2>/dev/null
        while read -r c; do
            c=`echo $c|tr -d '[:cntrl:]'`
            echo $c|egrep '^-' >/dev/null && continue
            _runjob $i "$c"
            echo "Ran $c;" >> $LOG
            i=`expr $i + 1`
            sleep 2
        done;;
    -submit)
        rm -f .jobs-suite.tmp 2>/dev/null
        while read -r c; do
            echo $c|egrep '^-' >/dev/null && continue
            eval "echo $c" >> .jobs-suite.tmp
        done
        scp "$0" "$USER@$RMHOST:$SCR"
        scp ".jobs-suite.tmp" $USER@$RMHOST:".jobs-suite.tmp"
        ssh -n -l $USER $RMHOST "sh $SCR -runall < .jobs-suite.tmp";;
    *)
        echo "Syntax:"
        echo "  -cls"
        echo "  -run JOB-NUMBER 'JOB-COMMAND'"
        echo "  -runall < FILE-WITH-COMMANDS"
        echo "  -submit < FILE-WITH-COMMANDS"
        echo "Used env. vars: \$USER (login), \$RMHOST (hostname)"
        exit 1;;
esac
:

четверг, 2 марта 2017 г.

Redirect file descriptors in Python for current and children processes

This is a Python trick with supressing of console output. Actually implementation does not supresses anything, but maps (dups) "source" file descriptors to "destination" file descriptors, so you can continue to use already existing descriptor, but it will point out to another one. "Another one" are got from list, but if dstfd list in lesser than srcfd list, then missing will be filled with "devnull", which allows us to suppress console output - this is the one way to use this class.

It is OS independent and was tested in Linux, Windows. Such "magia" happens only in context, after exit from context old values of descriptors will be restored. Note, that this works for children process too, sure. So, if you start some child process, you can "mask" its descriptors (redirect them somewhere).

import os, sys
from itertools import zip_longest

class FdRedir:
    def __init__(self, srcfd, dstfd):
        self.src = srcfd
        self.dst = dstfd
        self.null = os.open(os.devnull, os.O_RDWR)
        self.dups = [os.dup(f) for f in self.src]
    def __enter__(self):
        homo = zip_longest(self.src, self.dst, fillvalue=self.null)
        for t, f in homo:
            os.dup2(f, t)
        return self
    def __exit__(self, xtyp, xval, xtb):
        iso = zip(self.dups, self.src)
        for f, t in iso:
            os.dup2(f, t)
        for f in self.dups:
            os.close(f)
        os.close(self.null)

# Example of usage

with FdRedir((1, 2), ()) as fr:
    print('hidden')
    print('hidden', file=sys.stderr)
    with FdRedir([1], [fr.dups[0]]):
        print('visible tmp')
        print('hidden', file=sys.stderr)
    print('hidden')
print('visible end')

will print

visible tmp
visible end

which is expected: we suppress output to stdout, stderr descriptors in first context, then temporary allow stdout (by map it to fr.dups[0]).