вторник, 31 мая 2011 г.

Multithread queue

This C module is similar to Python Queue - for multithreading programs. It is based on POSIX Threads and BSD queue.h.
  • qitem_t is the queue element (holds void* of data)
  • queue_t is the main struct
  • qerror_t is the enumeration of errors' codes
  • create_queue() is used for creation with maximal elements count limit
  • delete_queue() is the destructor
  • clear_queue() deletes all elements (last arg determines how to freeing elements)
  • put_to_queue() put element
  • get_from_queue() get (and remove) one element if available
  • get_all_from_queue() get all (and remove them) elements from queue

See the code:

#include <assert.h>
#include <stdlib.h>
#include "mtqueue.h"
#ifdef HAVE_LEAKDEBUG
#  include "cMemDbg.h"
#endif
extern void panic(const char *msg);
queue_t*
create_queue(int maxlen) {
        queue_t* q = (queue_t*)malloc(sizeof (queue_t));
        if (!q)
                goto alloc_error;
        SIMPLEQ_INIT(&q->head);
        q->len = 0;
        pthread_mutex_init(&q->qlock, NULL);
        q->maxlen = maxlen;
        return (q);
alloc_error:
    panic("Can not allocate queue");
    return (NULL);
}
void
delete_queue(queue_t *q, void (*delete_data)(void*)) {
        if (q) {
                clear_queue(q, delete_data); // delete all items from queue
                pthread_mutex_destroy(&q->qlock);
                free(q); // free queue
        }
}
void
clear_queue(queue_t *q, void (*delete_data)(void*)) {
        qitem_t *itemp = NULL;
        assert(q);
        if (lock_queue(q)) {
                panic("Can not lock the queue qlock");
                return;
        }
        while (!SIMPLEQ_EMPTY(&q->head)) {
                itemp = SIMPLEQ_FIRST(&q->head);
                if (itemp) {
                        if (delete_data && itemp->data)
                                delete_data(itemp->data);
                        free(itemp);
                }
                SIMPLEQ_REMOVE_HEAD(&q->head, items);
        }
        q->len = 0;
        unlock_queue(q);
}
qerror_t
put_to_queue(queue_t *q, void *data) {
        qerror_t result = (qerror_t)0;
        qitem_t *item = NULL;
        lock_queue(q);
        if (q->len == q->maxlen) {
                result = EQUEUEFULL;
                goto unlock;
        }
        item = (qitem_t*)malloc(sizeof (qitem_t));
        if (!item) {
                result = EQUEUEMEMORY;
                goto unlock;
        }
        item->data = data;
        SIMPLEQ_INSERT_TAIL(&q->head, item, items);
        q->len++;
unlock:
        unlock_queue(q);
        return (result);
}
qerror_t
get_from_queue(queue_t *q, void **dataptr) {
        qerror_t result = (qerror_t)0;
        qitem_t *item = NULL;
        lock_queue(q);
        if (SIMPLEQ_EMPTY(&q->head)) {
                result = EQUEUEEMPTY;
                goto unlock;
        }
        item = SIMPLEQ_FIRST(&q->head);
        *dataptr = item->data;
        SIMPLEQ_REMOVE_HEAD(&q->head, items);
        q->len--;
        free(item);
unlock:
        unlock_queue(q);
        return (result);
}
qerror_t
get_all_from_queue(queue_t *q, void **dataptr, int *num) {
        int i;
        qerror_t result = (qerror_t)0;
        qitem_t *item = NULL;
        void **all = NULL;
        lock_queue(q);
        if (SIMPLEQ_EMPTY(&q->head)) {
                result = EQUEUEEMPTY;
                goto unlock;
        }
        assert(q->len);
        all = malloc(q->len * sizeof (void*));
        if (!all) {
                result = EQUEUEMEMORY;
                panic("Can not allocate result items for get_all_from_queue()");
                goto unlock;
        }
        for (i=0; i<q->len; i++) {
                item = SIMPLEQ_FIRST(&q->head);
                //assert(item);
                if (!item) break;
                all[i] = item->data;
                SIMPLEQ_REMOVE_HEAD(&q->head, items);
                free(item);
        }
        *num = q->len;
        *dataptr = all;
        q->len = 0;
unlock:
        unlock_queue(q);
        return (result);
}

Here is .h file:

#ifndef _MTQUEUE_H
#define _MTQUEUE_H
#include <pthread.h>
#include <queue.h>
typedef struct qitem_t {
        void *data;
        SIMPLEQ_ENTRY(qitem_t) items; /* links */
} qitem_t;
typedef SIMPLEQ_HEAD(qhead_t, qitem_t) qhead_t;
typedef struct queue_t {
        qhead_t head;
        int len;
        int maxlen;
        pthread_mutex_t qlock;
} queue_t;
typedef enum qerror_t {
        EQUEUEFULL = 1,
        EQUEUEEMPTY,
        EQUEUEMEMORY
} qerror_t;
#define FREE_QDATA free
#define lock_queue(q) pthread_mutex_lock(&(q)->qlock)
#define unlock_queue(q) pthread_mutex_unlock(&(q)->qlock)
queue_t* create_queue(int maxlen);
void delete_queue(queue_t *q, void (*delete_data)(void*));
qerror_t put_to_queue(queue_t *q, void *data);
qerror_t get_from_queue(queue_t *q, void **dataptr);
qerror_t get_all_from_queue(queue_t *q, void **dataptr, int *num);
void clear_queue(queue_t *q, void (*delete_data)(void*));
#endif /*!_MTQUEUE_H*/

And simple test:

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include "mtqueue.h"
#include "delays.h"
#define NCICLES 200
queue_t *q = NULL;
void panic(char *msg){
        printf("PANIC: %s\n", msg);
        exit(1);
}
/* Thread of putting an elements to queue.
 */
void* thrput(void *arg) {
        int n;
        int *data;
        int i = *(int*)arg;
        while (i--) {
                printf("put #%d\n", i);
                n = NCICLES;
                while (n--) {
                        data = (int*)malloc(sizeof (int)); /* msg in heap! */
                        if (!data) {
                                panic("Can not allocate data for put");
                        }
                        *data = i*NCICLES + n;
                        while (put_to_queue(q, (void*)data)) {
                            printf("*** ERROR: queue is full!\n");
                            // wait 1 sec and try again
                            pthread_delay(1000);
                        }
                }
        }
}
void thrget1() {
        int i;
        int num = 0;
        void *data = NULL;
        int** idata;
        int prev = -1;
        while (1) {
                data = NULL;
                if (get_all_from_queue(q, &data, &num)) {
                        pthread_delay(2000);
                        if (!q->len)
                                break;
                        else
                                continue;
                }
                printf("get_all_from_queue num=%d\n", num);
                idata = (int**)data;
                for (i=0; i<num; i++) {
                        printf("got %d (prev=%d)\n", *idata[i], prev);
                        assert(prev==-1 || prev-*idata[i]==1);
                        prev = *idata[i];
                        free(idata[i]);
                }
                free(data);
        }
end:
        return;
}

void thrget2() {
        int prev = -1;
        int *data;
        while (1) {
                if (get_from_queue(q, (void**)&data)) {
                        pthread_delay(2000);
                        if (!q>len)
                                break;
                        else
                                continue;
                }
                printf("got %d (prev=%d)\n", *data, prev);
                assert(prev==-1 || prev-*data==1);
                prev = *data;
                free(data);
        }
}
void *thrget(void *arg) {
        thrget2(); // first, get all until NULL
        thrget1(); // then get one by one until NULL
}
void
main() {
        int n = 2;
        pthread_t thrput_id, thrget_id;
        q = create_queue(200); // size of queue
        pthread_create(&thrput_id, NULL, thrput, (void*)&n);
        pthread_create(&thrget_id, NULL, thrget, NULL);
        // wait finishing of the threads
        pthread_join(thrget_id, NULL);
        pthread_join(thrput_id, NULL);
        delete_queue(q, FREE_QDATA);
#ifdef HAVE_LEAKDEBUG
        PrintMemoryLeakInfo();
#endif
}
Tested on Linux, BSD, Windows succesfully.

Validators library

Small but powerfull library of validators for Python. Supports chaining and are based on exceptions raising (it's part of the "HMI Pybase" project, see Projects page):
# -*- coding: cp1251 -*-
from pybase.utils import safe, atof

def _avoid_value(hole):
    """Decorator - return validator which enable any value except
    hole"""
    def f(value):
        if value==hole:
            raise ValueError
        else:
            return value
    return f

class BaseValidator:
    """Base validation. Default behaviour is to `eat` any not None value:

    >>> bv = BaseValidator()
    >>> bv(0)
    0
    >>> bv('a')
    'a'
    >>> bv(None)
    Traceback (most recent call last):
        ...
    ValueError
    """
    def __init__(self, vfunc=_avoid_value(None), hint="", opts={}, **kw):
        """vfunc is validation function which must raise Exception
        (ValueError!) on invalid data and return casted
        value to it's domain. opts is additional options for
        validator, default is value to be returned instead of
        exception raised when is set"""
        self.opts = dict(opts)
        self.hint = hint or self.__class__.__name__
        if "default" in kw:
            self.vfunc = safe(vfunc, kw["default"])
        else:
            self.vfunc = vfunc
    def __call__(self, *a, **kw):
        return self.vfunc(*a, **kw)

class SetValidator(BaseValidator):
    """Validate that value is in set.
    Note, in_ __init__ arg must have __contains__ and __str__.

    >>> sv = SetValidator()
    >>> sv.hint
    'any'
    >>> sv(9)
    9
    >>> sv = SetValidator(in_=(1,2,3))
    >>> sv.hint
    'in (1, 2, 3)'
    >>> sv(3)
    3
    >>> sv(9)
    Traceback (most recent call last):
        ...
    ValueError
    >>> sv = SetValidator("available weight", (1,2,3))
    >>> sv.hint
    'available weight'
    >>> sv(3)
    3
    >>> sv = SetValidator(in_=(1,2), default='unknown')
    >>> sv(0)
    'unknown'
    >>> sv = SetValidator(in_=xrange(0,1000))
    >>> sv(1)
    1
    >>> sv(1)
    1
    """
    def __init__(self, hint=None, in_=None, **kw):
        self.in_ = in_
        hint = hint or ("any" if in_ is None else "in %s"%str(in_))
        def vfunc(value):
            if self.in_ is None or value in self.in_:
                return value
            else:
                raise ValueError
        BaseValidator.__init__(self, vfunc, hint, **kw)

class IntValidator(BaseValidator):
    """Validate that value is integer:
    >>> iv = IntValidator()
    >>> iv("9")
    9
    >>> iv("a")
    Traceback (most recent call last):
        ...
    ValueError: invalid literal for int() with base 10: 'a'
    >>> iv("a", 16)
    10
    """
    def __init__(self, hint=None, **kw):
        BaseValidator.__init__(self, int, hint, **kw)

class FloatValidator(BaseValidator):
    """Validate that value is float:
    >>> fv = FloatValidator("weight", default='no')
    >>> fv.hint
    'weight'
    >>> fv(9)
    9.0
    >>> fv("9.12")
    9.12
    >>> fv("9,12")
    9.12
    >>> fv("a")
    'no'
    """
    def __init__(self, hint=None, **kw):
        def vfunc(value):
            if isinstance(value, str):
                return atof(value)
            else:
                return float(value)
        BaseValidator.__init__(self, vfunc, hint, **kw)

class ChainValidator(BaseValidator):
    """Validators chaining: all validators pipes value. If exception occurs
    in piping then default of this validator will be returned or exception
    occurs (if default is not set):
    >>> sv = SetValidator(in_=(1,2,3,4))
    >>> sv('1')
    Traceback (most recent call last):
        ...
    ValueError
    >>> iv = IntValidator(default=4)
    >>> av = ChainValidator(and_=(iv, sv), default='what?')
    >>> av('1')
    1
    >>> av('2')
    2
    >>> av('10')
    'what?'
    >>> av('x')
    4
    >>> av.hint
    'IntValidator & in (1, 2, 3, 4)'
    >>> av = ChainValidator(and_=(iv, sv), hint="int list", opts={"something":123})
    >>> av.hint
    'int list'
    >>> av.opts["something"]
    123
    """
    def __init__(self, and_, hint=None, **kw):
        self.and_ = and_
        def vfunc(value):
            v = value
            for validator in and_:
                v = validator(v)
            return v
        if not hint:
            h = []
            for v in and_:
                h.append(str(v.hint))
            h = " & ".join(h)
            #if h: h = "[%s]"%h
        else:
            h = hint
        BaseValidator.__init__(self, vfunc, h, **kw)

if __name__=="__main__":
    import doctest
    doctest.testmod()

Here are examples of usage (and extending):
def IntSetValidator(hint, in_, *a, **kw):
    """Checks does int set contents a value"""
    iv = IntValidator()
    sv = SetValidator(in_, *a, **kw)
    return ChainValidator(hint=hint, and_=(iv, sv))

class IntRangeValidator(BaseValidator):
    "Checks does value is contained in 'a-b' range"
    def __init__(self, hint=None, **kw):
        def vfunc(value):
            """Should raise ValueError (ONLY) on invalid value, else
            returns parsed value"""
            try:
                rng = [int(x.strip()) for x in value.split('-')]
                if rng[1] <= rng[0]:
                    raise Exception
                else:
                    return rng
            except:
                raise ValueError
        BaseValidator.__init__(self, vfunc, hint, **kw)

TinyBSD installation

This is:
  • Live USB system
  • Installer of TinyBSD (MFS based FreeBSD 8) on 16Mb (of system partition) for embedded devices
(see http://code.google.com/p/tinybsd-inst/ for downloads).
Script 'build.my' is used for image creation and copy to USB flash (all information will be destroyed!). Image is 32Mb. Original 'tinybsd' script is modified to support lists of includes and excludes files from image (see 'tinybsd.patch') via 'tinybsd.nofiles', 'tinybsd.basefiles' (last was in original TinyBSD).
'kh_inst' is the project/conf (similar to 'firewall', 'vpn', so on). Installation is made throught rc.d mechanism: etc/rc.d/osinst script will be called at start time and will ask user to boot from USB with live FS or to install system to HDD.
/etc/instskel contents all configuration of installed system:
  • bsdlabel
  • csh.cshrc
  • fdisk
  • fstab
  • motd
  • nodes
  • profile
  • rc.conf
  • ttys
...all are standard except 'nodes'. See them to know how installer will create slices, partitions, FSs, run-time configs... As to 'nodes', it's only predefined values of IP addresses of predefined nodes, user can select IP by node's name (when install system) or to enter the value manually.
Installed OS additionally contents 2 preinstalled tools: mksh shell (small but usuable from MirBSD project) and tmux - BSD console multiplexer.

This installer was tested on LX-800 (AMD based board for embedded solutions).

понедельник, 30 мая 2011 г.

Skeleton files for VIM

Very simple script for auto-create of file content by it's extension:
" Use :Skel to apply skeleton (template) to empty new file (skeleton is
" dot-file from vimfiles/skel/dot.
function _Skel()
    " for any files
    norm 1G0
    let ext = expand('%:e')
    let dotfile = $VIM . '/vimfiles/skel/dot.' . ext
    exec 'r ' . dotfile
    norm 1Gdd
endfunction
function HSkel()
    " for .h files
    call _Skel()
    let HGUARG = toupper(expand('%:t:r'))
    exec 'silent %s/%FILE%/' . HGUARG . '/g'
    norm G-
endfunction
function Skel()
    let ext = expand('%:e')
    if ext == 'h'
        call HSkel()
    elseif ext == 'w'
        call _Skel()
    endif
endfunction
command! -buffer Skel :call Skel()

For example, I have in vimfiles/skel/dot.h:
/*
 * $Id$
 */

#ifndef _%FILE%_H
#define _%FILE%_H


#endif /* !_%FILE%_H*/

and after creation of some.h I call :Skel and get:
/*
 * $Id$
 */

#ifndef _SOME_H
#define _SOME_H


#endif /* !_SOME_H*/

Animated plot with matplotlib

This is the components of auto-scrolled plots with matplotlib:
class Plot:
    """Base attributes of an any plot"""
    def __init__(self,
            fig=None, # matplotlib Figure
            canvas=None, # matplotlib Canvas
            tkw=None, # Tk widget
            title="", #
            xdata=[], # x data
            ydata=[], # y data
            xmin=None, # left xlim:datetime (chages in scroll with set_xlim)
            xmax=None, # right xlim:datetime (chages in scroll with set_xlim)
            tdelta=None, # step (period of polling of sensors, etc.):timedelta
            ylabel="", # y label
            lnc="red", # line color
            lnpt="-", # line point type
            lnw=1, # line width
            intc="#707070", # internal rect. color
            outc="#9E9E9E", # outter rect. color
            ylimpad=10, # padding around y axis limit (ylim is +/- ylimpad)
            ):
        self.fig = fig
        self.canvas = canvas
        self.tkw = tkw
        self.title = title
        self.xdata = xdata
        self.ydata = ydata
        self.xmin = xmin
        self.xmax = xmax
        self.tdelta = tdelta
        self.xband = self.xmax - self.xmin # width of axe 0x (timedelta)
        self.ylabel = ylabel
        self.lnc = lnc
        self.lnpt = lnpt
        self.lnw = lnw
        self.intc = intc
        self.outc = outc
        self.ylimpad = ylimpad

        self.ymin = 0
        self.ymax = 0

class AnimPlot(Frame):
    """One-line plot in own frame"""
    # TODO: blit not used
    def __init__(self, master, app, blit=False, title="", npoints=60, ylabel="", lnc="red", lnpt="-", lnw=1, intc="#707070", outc="#9E9E9E", ylimpad=10, period=1, *a, **kw):
        Frame.__init__(self, master, *a, **kw)
        self.app = app

        xmin = self.app.begin
        xmax = xmin + timedelta(seconds=npoints*period) # period is secs
        tdelta = timedelta(seconds=period)

        self.p = Plot(title=title, ylabel=ylabel, lnc=lnc, lnpt=lnpt, lnw=lnw, intc=intc, outc=outc, xdata=deque(maxlen=npoints), ydata=deque(maxlen=npoints), xmin=xmin, xmax=xmax, tdelta=tdelta, ylimpad=ylimpad)
        self.blit = blit # use double-buffering?
        self._create_matplot()
        self.p.tkw.pack()

    def _create_matplot(self):
        """Creates of matplotlib plot and changes
        self.p"""

        # calculate plot size in inches; dpi is the pixels count per inch
        dpi = 75.
        figh = self.winfo_screenheight()
        figh = figh/dpi - 1 # 1 for menu bar
        figh = figh/2 # two plots
        figw = figh*2.5 # aspect
        fig = Figure(figsize=(figw, figh), dpi=dpi)

        fig.patch.set_facecolor(self.p.outc)
        ax = fig.add_subplot(111)
        ax.patch.set_facecolor(self.p.intc)
        ax.set_title(self.p.title)
        ax.grid(True)
        ax.set_ylabel(self.p.ylabel)

        ax.plot_date([], [], self.p.lnpt, color=self.p.lnc, lw=self.p.lnw)
        ax.set_xlim(mdates.date2num(self.p.xmin), mdates.date2num(self.p.xmax))

        ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M:%S', utils.tzinfo()))
        ax.fmt_xdata = mdates.DateFormatter('%H:%M:%S', utils.tzinfo())

        canvas = FigureCanvas(fig, master=self)
        tkw = canvas.get_tk_widget()
        tkw.config(relief=FLAT, bd=0, takefocus=0, highlightthickness=0)
        self.p.fig = fig
        self.p.canvas = canvas
        self.p.tkw = tkw

    def _redraw(self):
        """Redraws plot with data from self.p automatically after append of the new data"""
        ax = self.p.fig.axes[0]
        line = ax.lines[0]

        line.set_xdata(self.p.xdata)
        line.set_ydata(self.p.ydata)

        x1 = self.p.xdata[-1]
        tm0 = mdates.num2date(x1) - self.p.xband + self.p.tdelta
        x0 = mdates.date2num(tm0)
        ax.set_xlim(x0, x1)

        y0 = 0
        y1 = self.p.ymax
        ax.set_ylim(y0 - self.p.ylimpad, y1 + self.p.ylimpad)
        self.p.canvas.draw()

    def appendy(self, y):
        tm = utils.now()
        tm = mdates.date2num(tm)
        self.p.xdata.append(tm)
        self.p.ymax = max(self.p.ymax, y)
        self.p.ydata.append(y)
        self._redraw()
Here is an example of usage:
aplt1 = AnimPlot(self.pltfr, app=self.app, name="tpm200plot1", lnc="red", lnw=2, title=u"Sensor PV1", ylabel=u"C", period=3)
aplt1.pack(side=TOP, expand=YES, fill=Y)
# ...
def onpolling():
    val = read_sensor()
    aplt1.appendy(pv1)

Crossplatform software delays

This is the simple macros for different delays usuable on Unix and Windows:
#ifndef _DELAYS_H
#define _DELAYS_H

#include 
#include 

/* Converting between seconds and CPU ticks (clock_t):
 *      MS2TICKS() converts milliseconds to ticks
 *      US2TICKS() converts microseconds to ticks
 *      TICKS2MS() converts ticks to milliseconds
 *      TICKS2US() converts ticks to microseconds
 */
#define MS2TICKS(MS) ((MS)*(CLOCKS_PER_SEC)/1000)
#define US2TICKS(US) ((US)*(CLOCKS_PER_SEC)/1000000)
#define TICKS2MS(T)  ((T)/((CLOCKS_PER_SEC)*1000))
#define TICKS2US(T)  ((T)/((CLOCKS_PER_SEC)*1000000))

/*
 * delay(MS)         sleeping on MS milliseconds
 */
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32)
#  include 
#  define delay(MS) Sleep(MS)
#else
#  if defined(HAVE_USLEEP)
#    define delay(MS) usleep((MS)*1000)
#  elif defined(HAVE_SLEEP)
#    include 
#    define delay(MS) sleep((MS)/1000) /* granularity only ONE SECOND! */
#  else
#    error Can not found any sleep_clocks() implementation ideas
# endif
#endif

/*
 * pthread_delay(MS) sleeping thread on MS milliseconds
 */
#if defined(HAVE_PTHREAD_DELAY_NP)
#  define pthread_delay(MS) \
     do { \
             struct timespec __timespec={(MS)/1000,((MS)%1000)*1000000000}; \
             pthread_delay_np(&__timespec); \
     } \
     while(0)
#else
#  define pthread_delay(MS) delay(MS)
#endif

/*
 * unow(PNOW) store in var pointed by PNOW microseconds since Epoch
 */
#define unow(PNOW) \
{ \
        struct timeval __tv;  \
        gettimeofday(&__tv, NULL); \
        *(PNOW) = (__tv.tv_sec * 1000000) + __tv.tv_usec; \
}

#endif /*!_DELAYS_H*/
If HAVE_PTHREAD_DELAY_NP will be defined before including of this file, then pthread_delay() will be wrapper for pthread_delay_np() - see Pthreads for Win32 headers.

суббота, 28 мая 2011 г.

Bakefile: Makefile's generator

It's known problem - general way of making the project. There are many tools for this task: CMake, automake/autoconf, ant (usual for Java) and many-many other. But some of them use own build system and very few uses standard way of Makefile (i.e. generates Makefiles). This method is based on some input format which is translated to Makefile for target compiler. For example, automake/autoconf, Premake (with Lua scripting) and Bakefile (written in Python and is crossplatform, like Premake and many other). It's authors are Vaclav Slavik, Francesco Montorsi.

Bakefile uses XML as input format then generates standard Makefiles for the targets:

autoconf       Makefile.in for GNU Autoconf
borland        Makefile for Borland C++ and Borland make
dmars          Generic Makefile for Digital Mars C/C++
dmars_smake    Makefile for Digital Mars C/C++ with SMAKE
gnu            Makefile for GNU toolchain: GNU Make, GCC...
mingw          Makefile for MinGW: mingw32-make, MinGW...
msvc           Makefile for Visual C++ with Microsoft nmake
msvc6prj       Microsoft Visual C++ 6.0 project files
msevc4prj      Microsoft Embedded Visual C++ 4 project files
msvs2003prj    MS Visual Studio 2003 project files
msvs2005prj    MS Visual Studio 2005 project files
msvs2008prj    MS Visual Studio 2008 project files
suncc          GNU makefile for SunCC compiler
symbian        Symbian development files
watcom         Makefile for OpenWatcom C/C++
xcode2         Apple Xcode 2.4 project files

It's input file looks like:
<exe id="test_sockets">
        <cflags>-fpack-struct</cflags>
        <sources>test_sockets.c</sources>
        <sources>socks.c</sources>
        <sys-lib>pthread</sys-lib>
        <if cond="TOOLSET=='win32'">
            <sys-lib>wsock32</sys-lib>
        </if>
</exe>

This pie of code will build (via native Makefile sure) test_sockets binary from test_sockets.c and socks.c, with pthread library (available on Unix and MinGW too) and with wsock32 library only on Windows.

Another example of custom tag:
<include file="presets/simple.bkl">
    <define-tag name="lit" rules="action">
        <if cond="TOOLSET=='unix'">
            <command></command>pyweb.py -s $(attributes['web']).w
            <command></command>rst2html.py $(attributes['web']).rst $(attributes['web']).html
        </if>
        <if cond="TOOLSET=='win32'">
            <command></command>cmd /C pyweb.py -s $(attributes['web']).w
            <command></command>cmd /C rst2html.py $(attributes['web']).rst $(attributes['web']).html
        </if>
    </define-tag>

    <action id="lit">
        <lit web="maintest">
        <lit web="socks">
    </lit></lit></action>
</include>

This snippet will weave and tangle "literated" sources (with pyweb tool): maintest.w and socks.w but in different manners under Windows and Unix.
All of these snippets should be in <makefile> tag! When you use some tag, see documentation on tag (what outer tags it expect).

Calling Gimp via shell

This Python class is a wrapper for Gimp and use shell to communicate. It's part of the "HMI pybase" project, see on Project's page. It supports batch mode, rc file (config), session cleaning. Example of usage:
def gen_gimprc(gfig_dir):
    """Contents of rc file: setup directory of GFig plugin files
    """
    ret = u"(gfig-path \"%s\")"%gfid_dir.replace('\\', '\\\\') # should be (gfig-path "xx\\xx")
    return ret

...

b = Gimpsh(gimp_exe_path, locals())
gimprc = gen_gimprc(gfid_dir)
b.append((u"'(sym-new \"%(fpath)s.%(fmt)s\" %(indexed)d %(width)d %(height)d)'",), gimprc=gimprc)
try:
    # call some Scheme script sym-new with pre-setup rc file
    b.execall()
except Exception, x:
    print "Error!"
Here are the code:
import tempfile
import sys
import os
from pybase import utils

class Session(list):
    """list of jobs that ran on the same Gimp instance
    """
#    def __unicode__(self):
#        s = u"%s (%s)"%(super(Session, self).__str__(), self._opts)
#        return s

    _tmpfiles = [] # paths of temp. files - STATIC!
    def __init__(self, *a):
        super(Session, self).__init__(*a)
        self._opts = []
        self._args = []

    def __del__(self):
        # remove all temporary files
        for p in Session._tmpfiles:
            try:
                os.unlink(p)
            except:
                pass

    def copy(self):
        """Independed copy"""
        items = self[:]
        cpy = Session(items)
        cpy._opts = self._opts
        cpy._args = self._args
        return cpy

    def set_opt(self, opt, value=None):
        """option opt (-x|--x), value may be used or not. In _opts will be
        '-x'|'-x xxx'|'--x'|'--x=xxx'
        """
        if value and sys.platform=="win32":
            value = utils.quote_win32(value)
        if opt.startswith("--"):
            # long option
            f = u"%s=%s"%(opt,value) if value else opt
        else:
            f = u"%s %s"%(opt,value) if value else opt
        self._opts.append(f)

    @property
    def gimprc(self):
        raise AttributeError
    @gimprc.setter
    def gimprc(self, value):
        """Exception raising is possible
        """
        if not value: return
        (fd, name) = tempfile.mkstemp(prefix="gimprc", text=True)
        Session._tmpfiles.append(name)
        with os.fdopen(fd, "w") as f:
            f.write(value.encode("utf8"))
            f.close()
            self.set_opt("-g", unicode(name))

    def set_args(self, value):
        """Exception raising is possible
        """
        if not value: return

        if sys.platform=="win32":
            value = [utils.quote_win32(v) for v in value]
        self._args = value

    @property
    def cmdline(self):
        return u"%s %s"%(u" ".join(self._opts), u" ".join(self._args))

class Gimpsh:
    """Call gimp via shell. Called command lines are appened
    by append(). It's one call or many with several instances.
    It uses jobs (runnable via -b .. mechanism), so several
    Scheme functions can be called on one Gimp instance
    """
    def __init__(self, gimp="gimp", vars=None):
        """gimp is the full path to gimp program.
        Vars is the dict of variables for formatting, can be
        changed on the fly.
        """
        self.gimp = gimp
        # list of lists: each list is ran in one task, last task is
        # interactive, other are not.
        self._bs = []
        self.vars = vars or {}

    def append(self, jobs=None, args=None, gimprc=None):
        """Append list of jobs, returns job-sequence id.
        kw are the sessions settings, like gimprc path.
        args are list of positional arguments.
        """
        if not jobs:
            jobs = []
        if type(jobs) not in (list, tuple):
            raise ValueError("jobs should be sequence")
        ret = len(self._bs)

        # create Session with attributes (for options)
        ses = Session(jobs) # list of jobs with session settings
        ses.gimprc = gimprc
        ses.set_args(args)

        self._bs.append(ses)
        return ret

    def _shsession(self, jobs, last):
        """Execute only jobs of one sequence on one gimp instance (one
        session), last is the opt of last sequence (last item in self._bs!)
        """
        if len(jobs) != 0:
            if not last:
                jobs = jobs.copy()
                # FIXME but final console message is not close, so until console is close
                # starting of next gimp is impossible, may be workaround with another
                # system() without waiting child PID
                jobs.append(u"'(gimp-quit 0)'")
                jobs.set_opt("-i")
            j = Gimpsh._prebatch(jobs, self.vars) # returns real list (not Session) of -b OPTs
        else:
            j = ""
        #jobs.set_opt("-s") # no splash window
        cl = u"%s %s %s" % (self.gimp, j, jobs.cmdline)
        utils.system(cl)

    def execone(self, jobsid):
        """Like _shsession but executed jobs are pointed by jobsid. If jobsid
        out of range, IndexError will be raised
        """
        self._shsession(self._bs[jobsid], jobsid==len(self._bs)-1)

    def execall(self, vars={}):
        """Executes all jobs sequence
        """
        for jobsid in xrange(len(self._bs)):
            self.execone(jobsid)

    @staticmethod
    def _prebatch(batches, vars={}):
        """Prepare batch for command line: quotes on win32, adds -b
        """
        if sys.platform ==  "win32":
            bs = [u"-b "+utils.quote_win32(b%vars) for b in batches]
        else:
            bs = [u"-b "+b%vars for b in batches]
        return u" ".join(bs)

Virtual root (Python module)

Virtual root is for sandbox - portable file system pie. May be used when app need portable FS pie (see "vroot" module in "HMI pybase" on Projects page):

VRoot:
  usr/
    share/
  lib
  ...etc...

And app have to access files under relative paths in VRoot (sandbox).

VRoot(mount_point) create virtual (sandbox) root with methods:
  • hpath(virtual_path) returns path on host
  • vpath(host_path, keep_host=False) returns virtual path, if not possible and keep_host, returns original host_path (None otherwise)
And function:
  • path2uri(path, base=None) returns URI, like a/b/c to be inserted to 'file:///HERE'

from os import path

class VRoot:
    """Use always unicode in path!
    """
    _mp = None

    def __init__(self, mp):
        "mp is the mount-point: abs. path on host"
        self.mount(mp)

    def hpath(self, vp):
        """Returns host path from virtual path:
        >>> vr = VRoot(u'/a/b')
        >>> vr.hpath(u'c///') == VRoot._fixed_abspath(u'/a/b/c')
        True
        >>> vr.hpath(u'/c///') == VRoot._fixed_abspath(u'/a/b/c')
        True
        >>> vr.hpath(u"../../../c/") == VRoot._fixed_abspath(u'/a/b/c')
        True
        >>> vr.hpath(u"/../c/") == VRoot._fixed_abspath(u'/a/b/c')
        True
        """
        vp = vp.replace(u"\\", u"/").split(u'/')
        vp = u'/'.join(x for x in vp if x and x!=u"..")
        return VRoot._fixed_abspath(path.join(self._mp, vp))

    def vpath(self, hp, keep_host=False):
        """Returns virtual path, relative to _mp. If keep_host, then
        returns original hp when hp is not in _mp, otherwise returns None:
        >>> vr = VRoot(u'/a/b/')
        >>> vr.vpath(u'/a/b/') == path.sep
        True
        >>> vr.vpath(u'/a/b/', True) == path.sep
        True
        >>> vr.vpath(u"/x///")
        >>> vr.vpath(u"/x/", True) == path.sep + u'x'
        True
        >>> vr = VRoot(u'/')
        >>> vr.vpath(u'/a/b/') == path.normpath(u"a/b")
        True
        """
        ap = VRoot._fixed_abspath(hp)
        ap = VRoot._fixed_relpath(ap, self._mp)
        if u".." in ap:
            if keep_host:
                return path.normpath(hp)
            else:
                return None
        elif ap == u".":
            return path.sep
        else:
            return ap

    @staticmethod
    def _pathroot(p):
        """Returns absolute path of root of path p:
        >>> VRoot._pathroot("/a/") == VRoot._fixed_abspath("/")
        True
        >>> VRoot._pathroot(r"d:\\x\\y\\\\")
        'd:\\\\'
        """
        p_drive = path.splitdrive(p)[0]
        if p_drive:
            # if there is the drive, absroot will be ''
            absroot = p_drive + path.sep
        else:
            # else will be default system root
            absroot = VRoot._fixed_abspath(u"/")
        return absroot

    @staticmethod
    def _fixed_abspath(p):
        """Issue: not normalized case"""
        return path.normcase(path.abspath(p))

    @staticmethod
    def _fixed_relpath(p, base):
        """Issue: when base is root, p is prefixed by '..':
        >>> VRoot._fixed_relpath('/a/b/c/', '/') == r'a\\b\\c'
        True
        >>> VRoot._fixed_relpath(r'd:\\x\\y', r'd:\\\\') == path.normpath("x/y")
        True
        """
        base_drive = path.splitdrive(base)[0]
        if base_drive:
            # if there is the drive, absroot will be ''
            absroot = base_drive + path.sep
        else:
            # else will be default system root
            absroot = VRoot._fixed_abspath(u"/")
        if VRoot._fixed_abspath(base) == absroot:
            # if base is root
            return path.normpath(p).lstrip(u".." + absroot)
        else:
            return path.relpath(p, base)

    def mount(self, mp):
        self._mp = VRoot._fixed_abspath(mp)

def path2uri(p, base=None):
    """Path to uri, but without 'file:///'! See:
    >>> path2uri('/a/b/c///')
    u'a/b/c'
    >>> path2uri(r'd:\\x\\y')
    u'x/y'
    >>> path2uri(u'/x/y/z', '/x')
    u'y/z'
    """
    return VRoot(base or VRoot._pathroot(p)).vpath(p).replace('\\', '/')

if __name__ == "__main__":
    import doctest
    doctest.testmod()
    #print VRoot._pathroot(r'd:\\\\x\y')

Multithreading console

There are several Unix programes with this feature: user can edit input line since program raise messages to console:
Message 3
Message 2
Message 1
enter name$ Alejandr_
It's implementation for Windows is simple (see Module "mtinput" in HMI pybase project on Projects page):
# Multi-thread console input() via class MTInput
import sys
import msvcrt
import string
from threading import Lock

# FIXME Needs Win32 module msvcrt! Under Linux is possible to use
# tty (but not on FAT FS system like in Linpac controllers!)
PROMPT = ">>>"
def normstr(s):
    """Interpret backspaces in string s,
    returns normal string
    """
    rs = reversed(s)
    res = []
    skip = 0
    for ch in rs:
        if ord(ch) == 8:
            skip += 1
            continue
        elif skip:
            skip -= 1
            continue
        else:
            res.append(ch)
    return u"".join(reversed(res))

class MTInput:
    """Multi-thread input line. Can be used like:

    import time
    import thread

    inputer = MTInput()
    def f():
    while 1:
        print '\ntest.......'
        inputer.update()
        time.sleep(2)
    thread.start_new_thread(f, ())
    while 1:
        inp = inputer.input()
        print 'ENTERED:', inp
        if inp == ".":
            break
        else:
            # ... process ...
    """
    def __init__(self, stdout=None, prompt=PROMPT):
        self._stdout = stdout or sys.stdout
        self._enc = getattr(self._stdout, "encoding", "ascii")
        self._prompt = prompt
        self._buf = []
        self._lock = Lock()

    def update(self):
        with self._lock:
            buf = u"".join(self._buf)
            try:
                s = u"%s %s" % (self._prompt, buf)
                encs = s.encode(self._enc)
                self._stdout.write(encs)
                self._stdout.flush()
            except:
                pass

    def input(self, prompt=PROMPT):
        with self._lock:
            self._buf = []
            self._prompt = prompt
        self._stdout.write(self._prompt + " ")
        self._stdout.flush()
        while True:
            ch = msvcrt.getwche()
            if ch in "\r\n":
                with self._lock:
                    buf = u"".join(self._buf)
                    self._buf = []
                buf = normstr(buf)
                return buf
            else:
                with self._lock:
                    self._buf.append(ch)

GFig GIMP 2.6 files parsing/rendering

See modules "gfig" from "HMI pybase" project on Projects page.

Parsing
GFig files are very simple vector format in Gimp 2.x used by GFig plugin for shapes creation. It looks like this:

GFIG Version 0.1
Name: Simly\040face
Version: 0.000000
ObjCount: 1
GridSpacing: 30
GridType: RECT_GRID
DrawGrid: FALSE
Snap2Grid: FALSE
LockOnGrid: FALSE
ShowControl: TRUE
66 150
66 200

Parses only KNOWN SHAPES, all other are ignored. Also parses styles and extra parameters. Suppose fixed structure of file: lines are not joined! Known shapes are:
  • line
  • rectangle
  • circle
  • ellipse
  • poly
  • star
  • spiral
  • bezier
  • arc

Rendering
GFig GIMP 2.6 files renders with Tkinter (on canvas). Supports:
  • lines
  • circles
  • ellipses
  • rectangles
  • polygons
  • arcs
  • old style spirals (via lines)
and it's outline-, fill- colors, line width; also polygon has style ruby (recreate canvas object when changed - interesting effect with smooth=1, ruby=True). Not supported: Bezier, spiral, stars. All unknown objects are ignored.

Transforming of objects is supported: move, scale, rotate. It's possible to transform only one object or any selected objects (set) of gfig-file. First, shape class methods are used, for second, GFigRender methods are used. GFigRender transformation methods has arg shapes - shapes to be transformed. Also, you can select GFigRender shapes by find_shapes() method and process it (transform, configure, for ex.). Finding use classname of shape ("CanvasArc", "CanvasEllipse", etc) or any option (Tk cget, gfig option) in **kw form, or name of shape. Usually shapes don't have names (gfig-extension feature), but user can name each shape in order from left-to-right and top-to-bottom (canvas location) with method name_shapes(). find_shapes() returns list, ifind_shapes() - iterator.

Each shape can have label - text short string. Technically, label is created on first labelconfigure(), each other only changes label options. Label is placed by options: side (N|NW|CENTER...) and standard Tk anchor option; so side specified corner (about shape bbox) or center of bbox

Contribs
Thanks for creators of Gimp GFig:
  • Andy Thomas
  • Spencer Kimball
  • Peter Mattis

четверг, 26 мая 2011 г.

Wireless configuration in Windows Vista

Change in registry branch "HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\ndisuio" value "Start" to 2. Then start services "Wired autoconfig", "WLAN Autoconfig" (first set to "Auto" starting mode - for autoconnecting - and now start them manually). If they doesn't start then see "Dependencies" tab and turn-on - to auto start mode all sub-services) and try again.
After this you can try to find a nets and to connect to one of them (see names of found networks). To connect you need to enter SSID and key phrase - see them in your Wireless Router config :)

See also:

вторник, 24 мая 2011 г.

Linux box on Windows

It's frequent task for programmer: developing of application for alien OS. If you use Windows as host system and need Linux box, there are several free solutions: VmWare Player, VirtualBox, QEmu for your pleasure :)

But after done with virtual machine, you should to create some "bridge" to data exchange with guest OS (ssh/scp, samba/nfs, ftp, shared folder...). When you use VirtualBox, for example, you should install additional software to guest OS. But there is simpler solution: qemu with virtual FAT folders.

Download from QEmu on Windows ZIP file with qemu (yes, it's portable, and is only about 4Mb in ZIP!). Uncompress.

Suppose, we need only developer's box: no X, only GNU toolchain.

Let's try Archlinux, see: www.archlinux.org (download "CORE" ISO image).
Then create image - space for VM:
qemu-img create linux.img 1G
1G is enough room. This will be done quickly.
Then let's boot from ArchLinux CD image:
qemu -m 256 -hda c:\vm\linux.img -cdrom c:\vm\archlinux-2010.05-core-i686.iso -boot d -L c:\qemu\bios
We suppose user uncompress qemu to c:\qemu and puts images to c:\vm. After boot you should install ArchLinux - it's easy, but long :)
OK, after installation you can run Linux box with sharing some folder from host OS. Create first some folder, for example, c:\vm\shared. Then run qemu:
qemu -m 256 -hda c:\vm\linux.img -L c:\qemu\bios -hdb fat:rw:/vm/shared
QEmu creates virtual drive (hdb) from host folder "c:\vm\shared", and you can mount it in Linux box:
su
mkdir /shared
mount -f vfat /dev/sdb1 /shared
"Read-Write" option in qemu is dangerous, so more preferable is to run qemu without "write" mode. For example, in current (0.13.0) version of Windows qemu build there is error: if you add some text to file in /shared, this file will be deleted (on host, yeh:). Use it for read-only access, and never modify shared folder on host when it is mounted in the guest OS.

Have a fun!