вторник, 31 мая 2011 г.

Multithread queue

This C module is similar to Python Queue - for multithreading programs. It is based on POSIX Threads and BSD queue.h.
  • qitem_t is the queue element (holds void* of data)
  • queue_t is the main struct
  • qerror_t is the enumeration of errors' codes
  • create_queue() is used for creation with maximal elements count limit
  • delete_queue() is the destructor
  • clear_queue() deletes all elements (last arg determines how to freeing elements)
  • put_to_queue() put element
  • get_from_queue() get (and remove) one element if available
  • get_all_from_queue() get all (and remove them) elements from queue

See the code:

#include <assert.h>
#include <stdlib.h>
#include "mtqueue.h"
#ifdef HAVE_LEAKDEBUG
#  include "cMemDbg.h"
#endif
extern void panic(const char *msg);
queue_t*
create_queue(int maxlen) {
        queue_t* q = (queue_t*)malloc(sizeof (queue_t));
        if (!q)
                goto alloc_error;
        SIMPLEQ_INIT(&q->head);
        q->len = 0;
        pthread_mutex_init(&q->qlock, NULL);
        q->maxlen = maxlen;
        return (q);
alloc_error:
    panic("Can not allocate queue");
    return (NULL);
}
void
delete_queue(queue_t *q, void (*delete_data)(void*)) {
        if (q) {
                clear_queue(q, delete_data); // delete all items from queue
                pthread_mutex_destroy(&q->qlock);
                free(q); // free queue
        }
}
void
clear_queue(queue_t *q, void (*delete_data)(void*)) {
        qitem_t *itemp = NULL;
        assert(q);
        if (lock_queue(q)) {
                panic("Can not lock the queue qlock");
                return;
        }
        while (!SIMPLEQ_EMPTY(&q->head)) {
                itemp = SIMPLEQ_FIRST(&q->head);
                if (itemp) {
                        if (delete_data && itemp->data)
                                delete_data(itemp->data);
                        free(itemp);
                }
                SIMPLEQ_REMOVE_HEAD(&q->head, items);
        }
        q->len = 0;
        unlock_queue(q);
}
qerror_t
put_to_queue(queue_t *q, void *data) {
        qerror_t result = (qerror_t)0;
        qitem_t *item = NULL;
        lock_queue(q);
        if (q->len == q->maxlen) {
                result = EQUEUEFULL;
                goto unlock;
        }
        item = (qitem_t*)malloc(sizeof (qitem_t));
        if (!item) {
                result = EQUEUEMEMORY;
                goto unlock;
        }
        item->data = data;
        SIMPLEQ_INSERT_TAIL(&q->head, item, items);
        q->len++;
unlock:
        unlock_queue(q);
        return (result);
}
qerror_t
get_from_queue(queue_t *q, void **dataptr) {
        qerror_t result = (qerror_t)0;
        qitem_t *item = NULL;
        lock_queue(q);
        if (SIMPLEQ_EMPTY(&q->head)) {
                result = EQUEUEEMPTY;
                goto unlock;
        }
        item = SIMPLEQ_FIRST(&q->head);
        *dataptr = item->data;
        SIMPLEQ_REMOVE_HEAD(&q->head, items);
        q->len--;
        free(item);
unlock:
        unlock_queue(q);
        return (result);
}
qerror_t
get_all_from_queue(queue_t *q, void **dataptr, int *num) {
        int i;
        qerror_t result = (qerror_t)0;
        qitem_t *item = NULL;
        void **all = NULL;
        lock_queue(q);
        if (SIMPLEQ_EMPTY(&q->head)) {
                result = EQUEUEEMPTY;
                goto unlock;
        }
        assert(q->len);
        all = malloc(q->len * sizeof (void*));
        if (!all) {
                result = EQUEUEMEMORY;
                panic("Can not allocate result items for get_all_from_queue()");
                goto unlock;
        }
        for (i=0; i<q->len; i++) {
                item = SIMPLEQ_FIRST(&q->head);
                //assert(item);
                if (!item) break;
                all[i] = item->data;
                SIMPLEQ_REMOVE_HEAD(&q->head, items);
                free(item);
        }
        *num = q->len;
        *dataptr = all;
        q->len = 0;
unlock:
        unlock_queue(q);
        return (result);
}

Here is .h file:

#ifndef _MTQUEUE_H
#define _MTQUEUE_H
#include <pthread.h>
#include <queue.h>
typedef struct qitem_t {
        void *data;
        SIMPLEQ_ENTRY(qitem_t) items; /* links */
} qitem_t;
typedef SIMPLEQ_HEAD(qhead_t, qitem_t) qhead_t;
typedef struct queue_t {
        qhead_t head;
        int len;
        int maxlen;
        pthread_mutex_t qlock;
} queue_t;
typedef enum qerror_t {
        EQUEUEFULL = 1,
        EQUEUEEMPTY,
        EQUEUEMEMORY
} qerror_t;
#define FREE_QDATA free
#define lock_queue(q) pthread_mutex_lock(&(q)->qlock)
#define unlock_queue(q) pthread_mutex_unlock(&(q)->qlock)
queue_t* create_queue(int maxlen);
void delete_queue(queue_t *q, void (*delete_data)(void*));
qerror_t put_to_queue(queue_t *q, void *data);
qerror_t get_from_queue(queue_t *q, void **dataptr);
qerror_t get_all_from_queue(queue_t *q, void **dataptr, int *num);
void clear_queue(queue_t *q, void (*delete_data)(void*));
#endif /*!_MTQUEUE_H*/

And simple test:

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include "mtqueue.h"
#include "delays.h"
#define NCICLES 200
queue_t *q = NULL;
void panic(char *msg){
        printf("PANIC: %s\n", msg);
        exit(1);
}
/* Thread of putting an elements to queue.
 */
void* thrput(void *arg) {
        int n;
        int *data;
        int i = *(int*)arg;
        while (i--) {
                printf("put #%d\n", i);
                n = NCICLES;
                while (n--) {
                        data = (int*)malloc(sizeof (int)); /* msg in heap! */
                        if (!data) {
                                panic("Can not allocate data for put");
                        }
                        *data = i*NCICLES + n;
                        while (put_to_queue(q, (void*)data)) {
                            printf("*** ERROR: queue is full!\n");
                            // wait 1 sec and try again
                            pthread_delay(1000);
                        }
                }
        }
}
void thrget1() {
        int i;
        int num = 0;
        void *data = NULL;
        int** idata;
        int prev = -1;
        while (1) {
                data = NULL;
                if (get_all_from_queue(q, &data, &num)) {
                        pthread_delay(2000);
                        if (!q->len)
                                break;
                        else
                                continue;
                }
                printf("get_all_from_queue num=%d\n", num);
                idata = (int**)data;
                for (i=0; i<num; i++) {
                        printf("got %d (prev=%d)\n", *idata[i], prev);
                        assert(prev==-1 || prev-*idata[i]==1);
                        prev = *idata[i];
                        free(idata[i]);
                }
                free(data);
        }
end:
        return;
}

void thrget2() {
        int prev = -1;
        int *data;
        while (1) {
                if (get_from_queue(q, (void**)&data)) {
                        pthread_delay(2000);
                        if (!q>len)
                                break;
                        else
                                continue;
                }
                printf("got %d (prev=%d)\n", *data, prev);
                assert(prev==-1 || prev-*data==1);
                prev = *data;
                free(data);
        }
}
void *thrget(void *arg) {
        thrget2(); // first, get all until NULL
        thrget1(); // then get one by one until NULL
}
void
main() {
        int n = 2;
        pthread_t thrput_id, thrget_id;
        q = create_queue(200); // size of queue
        pthread_create(&thrput_id, NULL, thrput, (void*)&n);
        pthread_create(&thrget_id, NULL, thrget, NULL);
        // wait finishing of the threads
        pthread_join(thrget_id, NULL);
        pthread_join(thrput_id, NULL);
        delete_queue(q, FREE_QDATA);
#ifdef HAVE_LEAKDEBUG
        PrintMemoryLeakInfo();
#endif
}
Tested on Linux, BSD, Windows succesfully.

Validators library

Small but powerfull library of validators for Python. Supports chaining and are based on exceptions raising (it's part of the "HMI Pybase" project, see Projects page):
# -*- coding: cp1251 -*-
from pybase.utils import safe, atof

def _avoid_value(hole):
    """Decorator - return validator which enable any value except
    hole"""
    def f(value):
        if value==hole:
            raise ValueError
        else:
            return value
    return f

class BaseValidator:
    """Base validation. Default behaviour is to `eat` any not None value:

    >>> bv = BaseValidator()
    >>> bv(0)
    0
    >>> bv('a')
    'a'
    >>> bv(None)
    Traceback (most recent call last):
        ...
    ValueError
    """
    def __init__(self, vfunc=_avoid_value(None), hint="", opts={}, **kw):
        """vfunc is validation function which must raise Exception
        (ValueError!) on invalid data and return casted
        value to it's domain. opts is additional options for
        validator, default is value to be returned instead of
        exception raised when is set"""
        self.opts = dict(opts)
        self.hint = hint or self.__class__.__name__
        if "default" in kw:
            self.vfunc = safe(vfunc, kw["default"])
        else:
            self.vfunc = vfunc
    def __call__(self, *a, **kw):
        return self.vfunc(*a, **kw)

class SetValidator(BaseValidator):
    """Validate that value is in set.
    Note, in_ __init__ arg must have __contains__ and __str__.

    >>> sv = SetValidator()
    >>> sv.hint
    'any'
    >>> sv(9)
    9
    >>> sv = SetValidator(in_=(1,2,3))
    >>> sv.hint
    'in (1, 2, 3)'
    >>> sv(3)
    3
    >>> sv(9)
    Traceback (most recent call last):
        ...
    ValueError
    >>> sv = SetValidator("available weight", (1,2,3))
    >>> sv.hint
    'available weight'
    >>> sv(3)
    3
    >>> sv = SetValidator(in_=(1,2), default='unknown')
    >>> sv(0)
    'unknown'
    >>> sv = SetValidator(in_=xrange(0,1000))
    >>> sv(1)
    1
    >>> sv(1)
    1
    """
    def __init__(self, hint=None, in_=None, **kw):
        self.in_ = in_
        hint = hint or ("any" if in_ is None else "in %s"%str(in_))
        def vfunc(value):
            if self.in_ is None or value in self.in_:
                return value
            else:
                raise ValueError
        BaseValidator.__init__(self, vfunc, hint, **kw)

class IntValidator(BaseValidator):
    """Validate that value is integer:
    >>> iv = IntValidator()
    >>> iv("9")
    9
    >>> iv("a")
    Traceback (most recent call last):
        ...
    ValueError: invalid literal for int() with base 10: 'a'
    >>> iv("a", 16)
    10
    """
    def __init__(self, hint=None, **kw):
        BaseValidator.__init__(self, int, hint, **kw)

class FloatValidator(BaseValidator):
    """Validate that value is float:
    >>> fv = FloatValidator("weight", default='no')
    >>> fv.hint
    'weight'
    >>> fv(9)
    9.0
    >>> fv("9.12")
    9.12
    >>> fv("9,12")
    9.12
    >>> fv("a")
    'no'
    """
    def __init__(self, hint=None, **kw):
        def vfunc(value):
            if isinstance(value, str):
                return atof(value)
            else:
                return float(value)
        BaseValidator.__init__(self, vfunc, hint, **kw)

class ChainValidator(BaseValidator):
    """Validators chaining: all validators pipes value. If exception occurs
    in piping then default of this validator will be returned or exception
    occurs (if default is not set):
    >>> sv = SetValidator(in_=(1,2,3,4))
    >>> sv('1')
    Traceback (most recent call last):
        ...
    ValueError
    >>> iv = IntValidator(default=4)
    >>> av = ChainValidator(and_=(iv, sv), default='what?')
    >>> av('1')
    1
    >>> av('2')
    2
    >>> av('10')
    'what?'
    >>> av('x')
    4
    >>> av.hint
    'IntValidator & in (1, 2, 3, 4)'
    >>> av = ChainValidator(and_=(iv, sv), hint="int list", opts={"something":123})
    >>> av.hint
    'int list'
    >>> av.opts["something"]
    123
    """
    def __init__(self, and_, hint=None, **kw):
        self.and_ = and_
        def vfunc(value):
            v = value
            for validator in and_:
                v = validator(v)
            return v
        if not hint:
            h = []
            for v in and_:
                h.append(str(v.hint))
            h = " & ".join(h)
            #if h: h = "[%s]"%h
        else:
            h = hint
        BaseValidator.__init__(self, vfunc, h, **kw)

if __name__=="__main__":
    import doctest
    doctest.testmod()

Here are examples of usage (and extending):
def IntSetValidator(hint, in_, *a, **kw):
    """Checks does int set contents a value"""
    iv = IntValidator()
    sv = SetValidator(in_, *a, **kw)
    return ChainValidator(hint=hint, and_=(iv, sv))

class IntRangeValidator(BaseValidator):
    "Checks does value is contained in 'a-b' range"
    def __init__(self, hint=None, **kw):
        def vfunc(value):
            """Should raise ValueError (ONLY) on invalid value, else
            returns parsed value"""
            try:
                rng = [int(x.strip()) for x in value.split('-')]
                if rng[1] <= rng[0]:
                    raise Exception
                else:
                    return rng
            except:
                raise ValueError
        BaseValidator.__init__(self, vfunc, hint, **kw)

TinyBSD installation

This is:
  • Live USB system
  • Installer of TinyBSD (MFS based FreeBSD 8) on 16Mb (of system partition) for embedded devices
(see http://code.google.com/p/tinybsd-inst/ for downloads).
Script 'build.my' is used for image creation and copy to USB flash (all information will be destroyed!). Image is 32Mb. Original 'tinybsd' script is modified to support lists of includes and excludes files from image (see 'tinybsd.patch') via 'tinybsd.nofiles', 'tinybsd.basefiles' (last was in original TinyBSD).
'kh_inst' is the project/conf (similar to 'firewall', 'vpn', so on). Installation is made throught rc.d mechanism: etc/rc.d/osinst script will be called at start time and will ask user to boot from USB with live FS or to install system to HDD.
/etc/instskel contents all configuration of installed system:
  • bsdlabel
  • csh.cshrc
  • fdisk
  • fstab
  • motd
  • nodes
  • profile
  • rc.conf
  • ttys
...all are standard except 'nodes'. See them to know how installer will create slices, partitions, FSs, run-time configs... As to 'nodes', it's only predefined values of IP addresses of predefined nodes, user can select IP by node's name (when install system) or to enter the value manually.
Installed OS additionally contents 2 preinstalled tools: mksh shell (small but usuable from MirBSD project) and tmux - BSD console multiplexer.

This installer was tested on LX-800 (AMD based board for embedded solutions).

понедельник, 30 мая 2011 г.

Skeleton files for VIM

Very simple script for auto-create of file content by it's extension:
" Use :Skel to apply skeleton (template) to empty new file (skeleton is
" dot-file from vimfiles/skel/dot.
function _Skel()
    " for any files
    norm 1G0
    let ext = expand('%:e')
    let dotfile = $VIM . '/vimfiles/skel/dot.' . ext
    exec 'r ' . dotfile
    norm 1Gdd
endfunction
function HSkel()
    " for .h files
    call _Skel()
    let HGUARG = toupper(expand('%:t:r'))
    exec 'silent %s/%FILE%/' . HGUARG . '/g'
    norm G-
endfunction
function Skel()
    let ext = expand('%:e')
    if ext == 'h'
        call HSkel()
    elseif ext == 'w'
        call _Skel()
    endif
endfunction
command! -buffer Skel :call Skel()

For example, I have in vimfiles/skel/dot.h:
/*
 * $Id$
 */

#ifndef _%FILE%_H
#define _%FILE%_H


#endif /* !_%FILE%_H*/

and after creation of some.h I call :Skel and get:
/*
 * $Id$
 */

#ifndef _SOME_H
#define _SOME_H


#endif /* !_SOME_H*/