From a77e3a63f004e6ee789fa05e4a5bbc333b1529f1 Mon Sep 17 00:00:00 2001 From: Lorry Tar Creator Date: Fri, 12 Sep 2014 14:16:22 +0000 Subject: Imported from /home/lorry/working-area/delta_python-packages_sysv-ipc-tarball/sysv_ipc-0.6.8.tar.gz. --- INSTALL | 4 + LICENSE | 24 + PKG-INFO | 37 + README | 12 + ReadMe.html | 1505 ++++++++++++++++++++++++++++++++++++ VERSION | 3 + common.c | 100 +++ common.h | 212 +++++ demo/ReadMe.txt | 54 ++ demo/SampleIpcConversation.png | Bin 0 -> 11909 bytes demo/cleanup.py | 29 + demo/conclusion.c | 139 ++++ demo/conclusion.py | 72 ++ demo/make_all.sh | 7 + demo/md5.c | 381 +++++++++ demo/md5.h | 91 +++ demo/params.txt | 18 + demo/premise.c | 211 +++++ demo/premise.py | 106 +++ demo/utils.c | 128 +++ demo/utils.h | 17 + demo/utils.py | 64 ++ demo/utils_for_2.py | 4 + demo/utils_for_3.py | 4 + demo2/ReadMe.txt | 41 + demo2/SampleIpcConversation.png | Bin 0 -> 11909 bytes demo2/cleanup.py | 17 + demo2/conclusion.py | 66 ++ demo2/params.txt | 4 + demo2/premise.py | 71 ++ demo2/utils.py | 41 + demo2/utils_for_2.py | 4 + demo2/utils_for_3.py | 4 + demo4/ReadMe.txt | 15 + demo4/child.py | 23 + demo4/parent.py | 46 ++ ftok_experiment.py | 50 ++ memory.c | 834 ++++++++++++++++++++ memory.h | 57 ++ mq.c | 667 ++++++++++++++++ mq.h | 79 ++ prober.py | 186 +++++ prober/probe_page_size.c | 22 + prober/semtimedop_test.c | 11 + prober/sniff_union_semun_defined.c | 12 + semaphore.c | 776 +++++++++++++++++++ semaphore.h | 53 ++ setup.py | 62 ++ sysv_ipc_module.c | 876 +++++++++++++++++++++ 49 files changed, 7239 insertions(+) create mode 100644 INSTALL create mode 100644 LICENSE create mode 100644 PKG-INFO create mode 100644 README create mode 100644 ReadMe.html create mode 100644 VERSION create mode 100644 common.c create mode 100644 common.h create mode 100644 demo/ReadMe.txt create mode 100644 demo/SampleIpcConversation.png create mode 100644 demo/cleanup.py create mode 100644 demo/conclusion.c create mode 100755 demo/conclusion.py create mode 100755 demo/make_all.sh create mode 100644 demo/md5.c create mode 100644 demo/md5.h create mode 100644 demo/params.txt create mode 100644 demo/premise.c create mode 100755 demo/premise.py create mode 100644 demo/utils.c create mode 100644 demo/utils.h create mode 100644 demo/utils.py create mode 100644 demo/utils_for_2.py create mode 100644 demo/utils_for_3.py create mode 100644 demo2/ReadMe.txt create mode 100644 demo2/SampleIpcConversation.png create mode 100755 demo2/cleanup.py create mode 100644 demo2/conclusion.py create mode 100644 demo2/params.txt create mode 100644 demo2/premise.py create mode 100644 demo2/utils.py create mode 100644 demo2/utils_for_2.py create mode 100644 demo2/utils_for_3.py create mode 100644 demo4/ReadMe.txt create mode 100644 demo4/child.py create mode 100644 demo4/parent.py create mode 100644 ftok_experiment.py create mode 100644 memory.c create mode 100644 memory.h create mode 100644 mq.c create mode 100644 mq.h create mode 100644 prober.py create mode 100644 prober/probe_page_size.c create mode 100644 prober/semtimedop_test.c create mode 100644 prober/sniff_union_semun_defined.c create mode 100644 semaphore.c create mode 100644 semaphore.h create mode 100644 setup.py create mode 100644 sysv_ipc_module.c diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..95eef8f --- /dev/null +++ b/INSTALL @@ -0,0 +1,4 @@ +To install, just use the normal setup.py routine: + +sudo python setup.py install + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c914ace --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2008, Philip Semanchuk +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of sysv_ipc nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY Philip Semanchuk ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL Philip Semanchuk BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..f11ee5f --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,37 @@ +Metadata-Version: 1.1 +Name: sysv_ipc +Version: 0.6.8 +Summary: System V IPC primitives (semaphores, shared memory and message queues) for Python +Home-page: http://semanchuk.com/philip/sysv_ipc/ +Author: Philip Semanchuk +Author-email: philip@semanchuk.com +License: http://creativecommons.org/licenses/BSD/ +Download-URL: http://semanchuk.com/philip/sysv_ipc/sysv_ipc-0.6.8.tar.gz +Description: Sysv_ipc gives Python programs access to System V semaphores, shared memory + and message queues. Most (all?) Unixes (including OS X) support System V IPC. + Windows+Cygwin 1.7 might also work. + + Sample code is included. + + sysv_ipc is free software (free as in speech and free as in beer) released + under a 3-clause BSD license. Complete licensing information is available in + the LICENSE file. + + You might also be interested in the similar POSIX IPC module at: + http://semanchuk.com/philip/posix_ipc/ + +Keywords: ipc inter-process communication semaphore shared memory shm message queue +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: POSIX :: BSD :: FreeBSD +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: POSIX :: SunOS/Solaris +Classifier: Operating System :: POSIX +Classifier: Operating System :: Unix +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 3 +Classifier: Topic :: Utilities diff --git a/README b/README new file mode 100644 index 0000000..48e9ee8 --- /dev/null +++ b/README @@ -0,0 +1,12 @@ +Sysv_ipc gives Python programs access to System V semaphores, shared memory +and message queues. Most (all?) Unixes (including OS X) support System V IPC. +Windows+Cygwin 1.7 might also work. + +Sample code is included. + +sysv_ipc is free software (free as in speech and free as in beer) released +under a 3-clause BSD license. Complete licensing information is available in +the LICENSE file. + +You might also be interested in the similar POSIX IPC module at: +http://semanchuk.com/philip/posix_ipc/ diff --git a/ReadMe.html b/ReadMe.html new file mode 100644 index 0000000..686eee6 --- /dev/null +++ b/ReadMe.html @@ -0,0 +1,1505 @@ + + + + + + + + + + System V IPC for Python - Semaphores, Shared Memory and Message Queues + + + + + + +

System V IPC for Python - Semaphores, Shared Memory and Message Queues

+ +
+ +
RSS +
+ +

This describes the sysv_ipc module which gives Python access +to System V inter-process semaphores, shared memory and message queues +on most (all?) *nix flavors. Examples include OS X, Linux, FreeBSD, +OpenSolaris 2008.11, +and AIX 5.2. +It might also work under Windows with a library like +Cygwin. +

+ +

It works with Python 2.4 – 3.x. +It's released +under a BSD license. +

+ +

You can download +sysv_ipc version 0.6.8 + +([md5 sum], +[sha1 sum]) + +which contains the source code, setup.py, installation instructions and +sample code. You can read about +all of the changes in this version. +

+ +

+You might also want to read +about some known bugs. +

+ +

You might be interested in the very similar module +posix_ipc +which provides Python access to POSIX IPC primitives. POSIX IPC is a little +easier to use than SysV IPC, but not all operating systems support it +completely. +

+ +

Module sysv_ipc

+ +

Jump to semaphores, +shared memory, or +message queues.

+ + +

Module Functions

+ +
+
attach(id, [address = None, [flags = 0]])
+
Attaches the (existing) shared memory that has the given id and + returns a new SharedMemory object. See + SharedMemory.attach() for details on the + address and flags parameters. + +

This method is useful only under fairly unusual circumstances. + You probably don't need it. +

+
+ +
ftok(path, id, [silence_warning = False])
+
Calls ftok(path, id). Note that + ftok() has limitations, and this + function will issue a warning to that effect unless + silence_warning is True. +
+ +
remove_semaphore(id)
+
Removes the semaphore with the given id.
+ +
remove_shared_memory(id)
+
Removes the shared memory with the given id.
+ +
remove_message_queue(id)
+
Removes the message queue with the given id.
+
+ + +

Module Constants

+ +
+
IPC_CREAT, IPC_EXCL and IPC_CREX
+
IPC_CREAT and IPC_EXCL are flags used when + creating IPC objects. They're + bitwise unique and can be ORed together. IPC_CREX is + shorthand for IPC_CREAT | IPC_EXCL. + +

When passed to an IPC object's constructor, IPC_CREAT indicates + that you want to create a new object or open an existing one. If you want + the call to fail if an object with that key already exists, specify + the IPC_EXCL flag, too. +

+
+ +
IPC_PRIVATE
+
This is a special value that can be passed in place of a key. It implies that + the IPC object should be available only to the creating process or its + child processes (e.g. those created with fork()). +
+ +
KEY_MIN and KEY_MAX
+
Denote the range of keys that this module accepts. Your OS might + limit keys to a smaller range depending on the typedef of + key_t. + +

Keys randomly generated by this module are in the range + 1 <e; key <e; SHRT_MAX. + That's type-safe unless your OS has a very bizarre + definition of key_t. +

+
+ +
SEMAPHORE_VALUE_MAX
+
The maximum value of a semaphore. +
+ +
PAGE_SIZE
+
The operating system's memory page size, in bytes. It's probably a good + idea to make shared memory segments some multiple of this size. +
+ +
SEMAPHORE_TIMEOUT_SUPPORTED
+
True if the platform supports timed semaphore waits, False otherwise.
+ +
SHM_RND
+
You probably don't need this, but it can be used when attaching shared + memory to force the address to be + rounded down to SHMLBA. See your system's man page for shmat() + for more information. +
+ +
SHM_HUGETLB, SHM_NORESERVE and SHM_REMAP
+
You probably don't need these. They're Linux-specific flags that can + be passed to the SharedMemory + constructor, or to the .attach() function in the case of + SHM_REMAP. See your system's man page for shmget() + and shmat() for more information. +
+
+ +

Module Errors

+ +

In addition to standard Python errors (e.g. ValueError), +this module raises custom errors. These errors cover +situations specific to IPC. +

+ +
+
Error
+
The base error class for all the custom errors in this module. This + error is occasionally raised on its own but you'll almost + always see a more specific error. +
+ +
InternalError
+
Indicates that something has gone very wrong in the module code. Please + report this to the maintainer. +
+ +
PermissionsError
+
Indicates that you've attempted something that the permissions on the + IPC object don't allow. +
+ +
ExistentialError
+
Indicates an error related to the existence or non-existence of + an IPC object. +
+ +
BusyError
+
Raised when a semaphore call to .P() or .Z() either times out + or would be forced to wait when its block attribute is False. +
+ +
NotAttachedError
+
Raised when a process attempts to read from or write to a shared memory + segment to which it is not attached. +
+
+ + +

The Semaphore Class

+ +

This is a handle to a semaphore.

+ +

Methods

+ +
+
Semaphore(key, [flags = 0, [mode = 0600, [initial_value = 0]]])
+
Creates a new semaphore or opens an existing one. + +

key must be None, + IPC_PRIVATE or + an integer > KEY_MIN and ≤ KEY_MAX. If the key + is None, the module chooses a random unused key. +

+ +

The flags specify whether you want to create a + new semaphore or open an existing one. +

+ +
    +
  • With flags set to the default of 0, the module attempts + to open an existing semaphore identified by key and raises + a ExistentialError if that semaphore doesn't exist. +
    • + +
  • With flags set to IPC_CREAT, the module + opens the semaphore identified by + key or creates a new + one if no such semaphore exists. Using IPC_CREAT by itself + is not recommended. (See Semaphore Initialization.) +
    • + +
  • With flags set to + IPC_CREX (IPC_CREAT | IPC_EXCL), + the module + creates a new semaphore identified by key. If a + semaphore with that key already exists, the call raises an + ExistentialError. + The initial_value is ignored unless + both of these flags are specified or + if the semaphore is read-only. +
    • +
+ +

When opening an existing semaphore, mode is ignored. +

+
+ +
acquire([timeout = None, [delta = 1]])
+
Waits (conditionally) until the semaphore's value is > 0 and then + returns, decrementing the semaphore. + +

The timeout (which can be a float) specifies how + many seconds this call should wait, if at all. +

+

The semantics of the timeout changed a little in + version 0.3. +

+ +
    +
  • A timeout of None (the default) + implies no time limit. The call will not return until its wait + condition is satisfied. +
    • + +
  • When timeout is 0, the call + raises a BusyError if it can't immediately + acquire the semaphore. Since it will + return immediately if not asked to wait, this can be + thought of as "non-blocking" mode. +
    • + +
  • When the timeout is > 0, the call + will wait no longer than timeout + seconds before either returning (having acquired the semaphore) + or raising a BusyError. +
    • +
+ +

When the call returns, the semaphore's value decreases by + delta + (or more precisely, abs(delta)) + which defaults to 1. +

+ +

On platforms that don't support the semtimedop() API call, + all timeouts (including zero) are treated as infinite. The call + will not return until its wait condition is satisfied. +

+ +

Most platforms provide semtimedop(). OS X is a + notable exception. The module's Boolean constant + SEMAPHORE_TIMEOUT_SUPPORTED + is True on platforms that support semtimedop(). +

+
+ + +
release([delta = 1])
+
+ Releases (increments) the semaphore. + +

The semaphore's value increases by delta + (or more precisely, abs(delta)) + which defaults to 1. +

+
+ +
P()
+
A synonym for .acquire() that takes the same parameters. + +

"P" stands for + prolaag or probeer te verlagen + (try to decrease), the original name given by + Edsger Dijkstra. +

+
+ +
V()
+
A synonym for .release() that takes the same parameters. + +

"V" stands for + verhoog (increase), the original name given by + Edsger Dijkstra. +

+
+ +
Z([timeout = None])
+
Blocks until zee zemaphore is zero. + +

Timeout has + the same meaning as described in .acquire(). +

+
+ +
remove()
+
+ Removes (deletes) the semaphore from the system. + +

As far as I can tell, the effect of deleting a semaphore that + other processes are still using is OS-dependent. Check your system's + man pages for semctl(IPC_RMID). +

+
+
+ +

Attributes

+ +
+
key (read-only)
+
The key passed in the call to the constructor.
+ +
id (read-only)
+
The id assigned to this semaphore by the OS.
+ +
value
+
The integer value of the semaphore.
+ +
undo
+
Defaults to False. + +

When True, operations that change the + semaphore's value will be undone (reversed) when + the process exits. Note that when a process exits, an undo operation + may imply that a semaphore's value should become negative or + exceed its maximum. + Behavior in this case is system-dependent, which means that + using this flag can make your code non-portable. +

+
+ +
block
+
+ Defaults to True, which means that calls to acquire() and + release() will not return + until their wait conditions are satisfied. + +

When False, these calls + will not block but will instead raise an error if they are unable + to return immediately. +

+
+ +
mode
+
The semaphore's permission bits. + +

Tip: the following Python code will display + the mode in octal:
+ print int(str(my_sem.mode), 8) +

+
+ +
uid
+
The semaphore's user id.
+ +
gid
+
The semaphore's group id.
+ +
cuid (read-only)
+
The semaphore creator's user id.
+ +
cgid (read-only)
+
The semaphore creator's group id.
+ +
last_pid (read-only)
+
The PID of the process that last called semop() (.P(), + .V() or .Z()) on this semaphore. +
+ +
waiting_for_nonzero (read-only)
+
The number of processes waiting for the value of the semaphore to become + non-zero (i.e. the number waiting in a call to .P()). +
+ +
waiting_for_zero (read-only)
+
The number of processes waiting for the value of the semaphore to become + zero (i.e. the number waiting in a call to .Z()). +
+ +
o_time (read-only)
+
The last time semop() (i.e. .P(), .V() or + .Z()) was called on this semaphore. +
+
+ +

Context Manager Support

+ +

These semaphores provide __enter__() and __exit__() +methods so they can be used in context managers. For instance -- +

+ +
+with sysv_ipc.Semaphore(name) as sem:
+    # Do something...
+
+ +

Entering the context acquires the semaphore, exiting the context releases + the semaphore. See demo4/child.py for a complete example. +

+ + +

The SharedMemory Class

+ +

This is a handle to a shared memory segment. +

+ + +

Methods

+ +
+
SharedMemory(key, [flags = 0, [mode = 0600, [size = 0 or PAGE_SIZE, [init_character = ' ']]]])
+
Creates a new shared memory segment or opens an existing one. + The memory is automatically attached. + +

key must be None, + IPC_PRIVATE or + an integer > 0 and ≤ KEY_MAX. If the key + is None, the module chooses a random unused key. +

+ +

The flags specify whether you want to create a + new shared memory segment or open an existing one. +

+ +
    +
  • With flags set to the + default of 0, the module attempts + to open an existing shared memory segment identified by + key and raises + a ExistentialError if it doesn't exist. +
    • + +
  • With flags set to IPC_CREAT, the module + opens the shared memory segment identified + by key or + creates a new one if no such segment exists. + Using IPC_CREAT by itself + is not recommended. (See Memory Initialization.) +
    • + +
  • With flags set to + IPC_CREX (IPC_CREAT | IPC_EXCL), + the module + creates a new shared memory segment identified by + key. If + a segment with that key already exists, the call raises + a ExistentialError. + +

    When both IPC_CREX is specified + and the caller has write permission, each byte in the new memory segment will be + initialized to the value of init_character. +

    +
    • +
+ +

The value of size depends on whether + one is opening an existing segment or creating a new one. +

+
    +
  • When opening an existing segment, size + must be ≤ the existing segment's size. Zero is + always valid. +
    • + +
  • When creating an new segment, + many (most? all?) operating systems insist on a size + > 0. + In addition, some round the size + up to the next multiple of PAGE_SIZE. +
    • +
+ +

This module supplies a default + size of PAGE_SIZE when + IPC_CREX is specified and 0 otherwise. +

+
+ +
attach([address = None, [flags = 0]])
+
+ Attaches this process to the shared memory. The memory must be attached + before calling .read() or .write(). Note that the + constructor automatically attaches the memory + so you won't need to call this method unless you explicitly detach it + and then want to use it again. + +

The address parameter allows one to specify (as a Python long) a memory + address at which to attach the segment. Passing None (the default) + is equivalent to passing NULL to shmat(). See that + function's man page for details. +

+ +

The flags are mostly only relevant if one specifies a specific address. + One exception is the flag SHM_RDONLY which, surprisingly, + attaches the segment read-only. +

+ +

Note that on some (and perhaps all) platforms, each call to .attach() + increments the system's "attached" count. Thus, if each call to + .attach() isn't paired with a call to .detach(), + the system's "attached" count for the shared memory segment will not + go to zero when the process exits. As a result, the shared memory + segment may not disappear even when its creator calls .remove() + and exits. +

+
+ +
detach()
+
Detaches this process from the shared memory.
+ +
read([byte_count = 0, [offset = 0]])
+
Reads up to byte_count bytes from the + shared memory segment starting at offset + and returns them as a string under Python 2 or as a bytes object + under Python 3. + +

If byte_count is zero (the default) the + entire buffer is returned. +

+ +

This method will never attempt to read past the end of the shared + memory segment, even when + offset + byte_count + exceeds the memory segment's size. In that case, the bytes + from offset to the end of the segment are returned. +

+
+ +
write(s, [offset = 0])
+
Writes the string s to the shared memory, + starting at offset. + +

At most n bytes will be written, where + n = the segment's size minus offset. +

+ +

The string may contain embedded NULL bytes ('\0'). +

+ +
remove()
+
Removes (destroys) the shared memory. Note that actual destruction of the + segment only occurs when all processes have detached. +
+ +
+ +

Attributes

+ +
+
key (read-only)
+
The key provided in the constructor.
+ +
id (read-only)
+
The id assigned to this semaphore by the OS.
+ +
size (read-only)
+
The size of the segment in bytes.
+ +
address (read-only)
+
The address of the segment as Python long.
+ +
attached (read-only)
+
If True, this segment is currently attached.
+ +
last_attach_time (read-only)
+
The last time a process attached this segment.
+ +
last_detach_time (read-only)
+
The last time a process detached this segment.
+ +
last_change_time (read-only)
+
The last time a process changed the uid, gid or mode on this segment.
+ +
creator_pid (read-only)
+
The PID of the process that created this segment.
+ +
last_pid (read-only)
+
The PID of the most last process to attach or detach this segment.
+ +
number_attached (read-only)
+
The number of processes attached to this segment.
+ +
uid
+
The segment's user id.
+ +
gid
+
The segment's group id.
+ +
mode
+
The shared memory's permission bits. + +

Tip: the following Python code will display + the mode in octal:
+ print int(str(my_mem.mode), 8) +

+
+ +
cuid (read-only)
+
The segment creator's user id.
+ +
cgid (read-only)
+
The segment creator's group id.
+
+ + +

The MessageQueue Class

+ +

This is a handle to a message queue.

+ +

Methods

+ +
+
MessageQueue(key, [flags = 0, [mode = 0600, [max_message_size = 2048]]])
+
Creates a new message queue or opens an existing one. + +

key must be None, + IPC_PRIVATE or + an integer > 0 and ≤ KEY_MAX. If the key + is None, the module chooses a random unused key. +

+ +

The flags specify whether you want to create a + new queue or open an existing one. +

+ +
    +
  • With flags set to the + default of 0, the module attempts + to open an existing message queue identified by + key and raises + a ExistentialError if it doesn't exist. +
    • + +
  • With flags set to IPC_CREAT, the module + opens the message queue identified + by key or + creates a new one if no such queue exists. +
    • + +
  • With flags set to + IPC_CREX (IPC_CREAT | IPC_EXCL), + the module + creates a new message queue identified by + key. If + a queue with that key already exists, the call raises + a ExistentialError. +
    • +
+ +

The max_message_size can be increased + from the default, but be aware of the issues discussed in + Message Queue Limits. +

+
+ +
send(message, [block = True, [type = 1]])
+
Puts a message on the queue. + +

The message string can contain embedded + NULLs (ASCII 0x00). +

+ +

The block flag specifies whether or + not the call should wait if the message can't be sent (if, for + example, the queue is full). When block + is False, the call will raise a BusyError if + the message can't be sent immediately. +

+ +

The type is + associated with the message and is relevant when calling + receive(). It must be > 0. +

+
+ +
receive([block = True, [type = 0]])
+
+ Receives a message from the queue, returning a tuple of + (message, type). Under Python 3, the message is a + bytes object. + +

The block flag specifies whether or + not the call should wait if there's no messages of the + specified type to retrieve. When block + is False, the call will raise a BusyError if + a message can't be received immediately. +

+ +

The type permits some control over + which messages are retrieved. +

+ +
    +
  • When type == 0, the call + returns the first message on the queue regardless of its + type. +
    • +
  • When type > 0, the call + returns the first message of that type. +
    • +
  • When type < 0, the call + returns the first message of the lowest type that is ≤ the + absolute value of type. +
    • +
+
+ +
remove()
+
Removes (deletes) the message queue.
+
+ +

Attributes

+ +
+
key (read-only)
+
The key provided in the constructor.
+ +
id (read-only)
+
The id assigned to this queue by the OS.
+ +
max_size
+
The maximum size of the queue in bytes. Only a process with + "appropriate privileges" can increase this value, and on some + systems even that won't work. See + Message Queue Limits for details. +
+ +
last_send_time (read-only)
+
The last time a message was placed on the queue.
+ +
last_receive_time (read-only)
+
The last time a message was received from the queue.
+ +
last_change_time (read-only)
+
The last time a process changed the queue's attributes.
+ +
last_send_pid (read-only)
+
The id of the most recent process to send a message.
+ +
last_receive_pid (read-only)
+
The id of the most recent process to receive a message.
+ +
current_messages (read-only)
+
The number of messages currently in the queue.
+ +
uid
+
The queue's user id.
+ +
gid
+
The queue's group id.
+ +
mode
+
The queue's permission bits. + +

Tip: the following Python code will display + the mode in octal:
+ print int(str(my_mem.mode), 8) +

+
+ +
cuid (read-only)
+
The queue creator's user id.
+ +
cgid (read-only)
+
The queue creator's group id.
+
+ + + +

Supported Features and Differences from SHM

+ +

This module is almost, but not quite, a superset of +shm. +Some of the additional features are the ability to override the block +flag on a per-call basis, the ability to change the semaphore's value +in increments > 1 when calling .P() and .V() +and exposure of sem_otime. +

+ +

Differences that might trip you up are listed below.

+ + + +

Usage Tips

+ +

Sample Code

+ +

This module comes with two demonstration apps. The first (in the +directory demo) shows how to use shared memory and semaphores. +The second (in the directory demo2) shows how to use +message queues. + + +

The Weakness of ftok()

+ +

+Most System V IPC sample code recommends ftok() for generating an +integer key that's more-or-less random. +It does not, however, guarantee that the key it generates is unused. If +ftok() gives your application a key that some other application is +already using, +your app is in trouble unless it has a reliable second mechanism for generating +a key. And if that's the case, why not just abandon ftok() and use the +second mechanism exclusively? +

+ +

This is the weakness of ftok() -- it isn't guaranteed to give you +what you want. The BSD +man page for ftok says it is "quite possible for the routine to +return duplicate keys". The term "quite possible" isn't quantified, but suppose +it means one-tenth of one percent. Who wants to have 1-in-1000 odds of a +catastrophic failure in their program, or even 1-in-10000? +

+ +

This module obviates the need for ftok() by generating random +keys for you. If your application can't use sysv_ipc's automatically +generated keys because it needs to know the key in advance, hardcoding a +random number like 123456 in your app might be no worse than using +ftok() and has the advantage of not hiding its limitations. +

+ +

This module provides ftok() in case you want to experiment with it. +However, to emphasize its weakness, this version of ftok() raises a +warning with every call unless you explicitly pass a flag to silence it. +

+ +

This package also provides ftok_experiment.py so that you can observe +how often ftok() generates duplicate keys on your system. +

+ + +

Semaphore Initialization

+ +

When a System V sempahore is created at the C API level, the OS is not required +to initialize the semaphore's value. (This per +the +SUSv3 standard for semget().) +Some (most? all?) operating systems initialize it to zero, but this behavior +is non-standard and therefore can't be relied upon. +

+ +

If sempahore creation happens in an predictable, orderly fashion, this isn't a +problem. But a +race condition arises when multiple processes vie to create/open the same semaphore. The +problem lies in the fact that when an application calls semget() with only +the IPC_CREAT flag, the caller can't tell whether or not he has +created a new semaphore or opened an existing one. +This makes it +difficult to create reliable code without using IPC_EXCL. +W. Richard Stevens' Unix Network Programming Volume 2 +calls this "a fatal flaw in the design of System V semaphores" (p 284). +

+ +

+For instance, imagine processes P1 and P2. They're executing the same code, +and that code intends to share a binary semaphore. +Consider the following sequence of events at the startup of P1 and P2 – +

+ +
    +
  1. P1 calls semget(IPC_CREAT) to create the semaphore S.
    2. +
  2. P2 calls semget(IPC_CREAT) to open S.
    3. +
  3. P1 initializes the semaphore's value to 1.
    4. +
  4. P1 calls acquire(), decrementing the value to 0.
    5. +
  5. P2, assuming S is a newly-created semaphore that needs to be initialized, + incorrectly sets the semaphore's value to 1.
    6. +
  6. P2 calls acquire(), decrementing the value to 0. Both processes + now think they own the lock.
    7. +
+ +

W. Richard Stevens' solution for this race condition is to check the value of +sem_otime (an element in the semid_ds struct that's +populated on the call to semctl(IPC_STAT) and which is exposed to +Python by this module) which +is initialized to zero when the semaphore is created and otherwise holds +the time of the last +call to semop() (which is called by P()/acquire(), +V()/release(), and Z()). +

+ +

In Python, each process would run something like this: +

+try:
+    sem = sysv_ipc.Semaphore(42, sysv_ipc.IPC_CREX)
+except sysv_ipc.ExistentialError:
+    # One of my peers created the semaphore already
+    sem = sysv_ipc.Semaphore(42)
+    # Waiting for that peer to do the first acquire or release
+    while not sem.o_time:
+        time.sleep(.1)
+else:
+    # Initializing sem.o_time to nonzero value
+    sem.release()
+# Now the semaphore is safe to use.
+
+ + +

Shared Memory Initialization

+ +

With shared memory, +using the IPC_CREAT flag without IPC_EXCL +is problematic unless you know the size of the segment +you're potentially opening. +

+ +

Why? Because when creating a new segment, +many (most? all?) operating systems demand a non-zero size. However, +when opening an existing segment, zero is the only guaranteed safe value +(again, assuming one doesn't know the size of the segment in advance). +Since IPC_CREAT +can open or create a segment, there's no safe value for the size under +this circumstance. +

+ +

As a (sort of) side note, the +SUSv3 +specification for shmget() says only that the size of a new +segment must not be less than "the system-imposed minimum". I +gather that at one time, some systems set the minimum at zero despite the +fact that it doesn't make much sense to create a zero-length shared memory +segment. I think most modern systems do the sensible thing and insist on +a minimum length of 1. +

+ + +

Message Queue Limits

+ +

Python programmers can usually remain blissfully ignorant of memory +allocation issues. Unfortunately, a combination of factors makes them +relevant when dealing with System V message queues. +

+ +

Some implementations impose extremely stingy limits. +For instance, many BSDish systems (OS X, FreeBSD, +NetBSD, and +OpenBSD) +limit queues to 2048 bytes. Note that that's the total +queue size, not the message size. Two 1k messages would fill the queue. +

+ +

Those limits can be very difficult to change. At best, +only privileged processes can increase the limit. At worst, the limit +is a kernel parameter and requires a kernel change via a tunable or +a recompile. +

+ +

This module can't figure out what the limits are, so +it can't cushion them or even report them to you. +On some systems the limits are expressed in header files, on others +they're available through kernel interfaces (like FreeBSD's sysctl). +Under OS X and to some extent OpenSolaris I can't figure out where they're +defined and what I report here is the result of experimentation and educated +guesses formed by Googling. +

+ +

The good news is that this module will still behave as advertised no +matter what these limits are. Nevertheless you might be surprised when a +call to .send() get stuck because a queue is full even though you've +only put 2048 bytes of messages in it. +

+ +

Here are the limits I've been able to find under my test operating +systems, ordered from best (most generous) to worst (most stingy). +This information was current as of 2009 when I wrote the +message queue code. It's getting pretty stale now. I hope the situation has +improved over the 2009 numbers I describe below. +

+ +

Under OpenSolaris 2008.05 each queue's maximum size defaults +to 64k. A privileged process (e.g. root) can change this through the +max_size attribute of a sysv_ipc.MessageQueue object. +I was able to increase it to 16M and successfully sent sixteen 1M messages to +the queue. +

+ +

Under Ubuntu 8.04 (and perhaps other Linuxes) each +queue's maximum size defaults to 16k. As with OpenSolaris, I was able to +increase this to 16M, but only for a privileged process. +

+ +

Under FreeBSD 7 and I think NetBSD and OpenBSD, each +queue's maximum size defaults to 2048 bytes. Furthermore, one can (as root) +set max_size to something larger and FreeBSD doesn't complain, but +it also ignores the change. +

+ +

OS X is the worst of the lot. Each queue is limited +to 2048 bytes and OS X silently ignores attempts to increase this (just like +FreeBSD). To add insult to injury, there appears to be no way to increase +this limit short of recompiling the kernel. +I'm guessing at this based on the +Darwin +message queue limits. +

+ +

If you want +to search for these limits on your operating system, the key constants are +MSGSEG, MSGSSZ, MSGTQL, MSGMNB, +MSGMNI and MSGMAX. Under BSD, sysctl kern.ipc +should tell you what you need to know and may allow you to change these +parameters. +

+ +

Nobody Likes a Mr. Messy

+ +

Semaphores and especially shared memory are a little different from most Python objects +and therefore require a little more care on the part of the programmer. When a +program creates a semaphore or shared memory object, it creates something that +resides outside of its own process, just like a file on a hard drive. It +won't go away when your process ends unless you explicitly remove it. +

+ +

In short, remember to clean up after yourself.

+ +

Consult Your Local man Pages

+ +

The sysv_ipc module is just a wrapper around your system's API. If your +system's implementation has quirks, the man pages for semget, semctl, semop +shmget, shmat, shmdt and shmctl will probably cover them. +

+ +

Interesting Tools

+ +

Many systems (although not some older versions of OS X) come +with ipcs and ipcrm. +The former shows existing shared memory, semaphores and message queues on your system and +the latter allows you to remove them. +

+ + +

Last But Not Least

+ +

For Pythonistas –

+ + +

Known Bugs

+ +

Bugs? My code never has bugs! There are, however, some suboptimal anomalies...

+ + + +

Version History

+ + + + +

Future Features/Changes

+ +

These are features that may or may not be added depending on technical +difficulty, user interest and so forth. +

+ + + +

I don't plan on adding support for semaphore sets.

+ + + + diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..fef123b --- /dev/null +++ b/VERSION @@ -0,0 +1,3 @@ +0.6.8 + + diff --git a/common.c b/common.c new file mode 100644 index 0000000..a55f99c --- /dev/null +++ b/common.c @@ -0,0 +1,100 @@ +#include "Python.h" +#include "structmember.h" + +#include "common.h" + +#include + +key_t +get_random_key(void) { + int key; + + /* ****************************************************************** + The inability to know the range of a key_t requires careful code here. + Remember that KEY_MIN and KEY_MAX refer only to the limits inherent in the + variable type I use internally when turning a key into a Python object and + vice versa. Those limits may exceed the operating system's limits of key_t. + + For instance, if key_t is typedef-ed as uint, I should generate a key + where 0 <= key <= UINT_MAX. + + Since I can't know what key_t is typedef-ed as, I take a conservative + approach and generate only keys where + 1 <= key <= SHRT_MAX. + + Such values will work if key_t is typedef-ed as a short, int, uint, + long or ulong. + ****************************************************************** */ + do { + // ref: http://www.c-faq.com/lib/randrange.html + key = ((int)((double)rand() / ((double)RAND_MAX + 1) * (SHRT_MAX - 1))) + 1; + } while (key == IPC_PRIVATE); + + return (key_t)key; +} + +#if PY_MAJOR_VERSION < 3 +PyObject * +py_int_or_long_from_ulong(unsigned long value) { + // Python ints are guaranteed to accept up to LONG_MAX. Anything + // larger needs to be a Python long. + if (value > LONG_MAX) + return PyLong_FromUnsignedLong(value); + else + return PyInt_FromLong(value); +} +#endif + + +int +convert_key_param(PyObject *py_key, void *converted_key) { + // Converts a PyObject into a key if possible. Returns 0 on failure. + // The converted_key param should point to a NoneableKey type. + // None is an acceptable key, in which case converted_key->is_none + // is non-zero and converted_key->value is undefined. + int rc = 0; + long key = 0; + + ((NoneableKey *)converted_key)->is_none = 0; + + if (py_key == Py_None) { + rc = 1; + ((NoneableKey *)converted_key)->is_none = 1; + } +#if PY_MAJOR_VERSION < 3 + else if (PyInt_Check(py_key)) { + rc = 1; + key = PyInt_AsLong(py_key); + } +#endif + else if (PyLong_Check(py_key)) { + rc = 1; + key = PyLong_AsLong(py_key); + if (PyErr_Occurred()) { + // This happens when the Python long is too big for a C long. + rc = 0; + PyErr_Format(PyExc_ValueError, + "Key must be between %ld (KEY_MIN) and %ld (KEY_MAX)", + KEY_MIN, KEY_MAX); + } + } + + if (rc) { + // Param is OK + if (! ((NoneableKey *)converted_key)->is_none) { + // It's not None; ensure it is in range + if ((key >= KEY_MIN) && (key <= KEY_MAX)) + ((NoneableKey *)converted_key)->value = (key_t)key; + else { + rc = 0; + PyErr_Format(PyExc_ValueError, + "Key must be between %ld (KEY_MIN) and %ld (KEY_MAX)", + KEY_MIN, KEY_MAX); + } + } + } + else + PyErr_SetString(PyExc_TypeError, "Key must be an integer or None"); + + return rc; +} diff --git a/common.h b/common.h new file mode 100644 index 0000000..89fa376 --- /dev/null +++ b/common.h @@ -0,0 +1,212 @@ +#include "Python.h" +#include "structmember.h" + +#if PY_MAJOR_VERSION >= 0x02050000 +#define PY_SSIZE_T_CLEAN +#define PY_STRING_LENGTH_MAX PY_SSIZE_T +#else +#define PY_STRING_LENGTH_MAX INT_MAX +#endif + +// define Py_TYPE for versions before Python 2.6 +#ifndef Py_TYPE +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#endif + +// define PyVarObject_HEAD_INIT for versions before Python 2.6 +#ifndef PyVarObject_HEAD_INIT +#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, +#endif + +/* In sys/ipc.h under OS X, I get stuck with the old, "do not use" version +of the ipc_perm struct. This is unavoidable because Python.h #undefs +_POSIX_C_SOURCE and _XOPEN_SOURCE. This causes cdefs.h to #define +__DARWIN_UNIX03 to 0 (implying the "default" compilation environment, as +opposed to "strict") and therefore I get the crappy, old version of the +struct. I don't think there's any way around this unless Python changes +its headers. +*/ +#include +#include +#include +#include + +#include "probe_results.h" + +/* Struct to contain a key which can be None */ +typedef struct { + int is_none; + key_t value; +} NoneableKey; + + +/* These identifiers are prefixed with SVIFP_ which stands for SysV Ipc +For Python. It's really just a random string of letters to prevent clashes +with other constants (as happens with SHM_SIZE on AIX). +*/ +enum GET_SET_IDENTIFIERS { + SVIFP_IPC_PERM_UID = 1, + SVIFP_IPC_PERM_GID, + SVIFP_IPC_PERM_CUID, + SVIFP_IPC_PERM_CGID, + SVIFP_IPC_PERM_MODE, + SVIFP_SEM_OTIME, + SVIFP_SHM_SIZE, + SVIFP_SHM_LAST_ATTACH_TIME, + SVIFP_SHM_LAST_DETACH_TIME, + SVIFP_SHM_LAST_CHANGE_TIME, + SVIFP_SHM_CREATOR_PID, + SVIFP_SHM_LAST_AT_DT_PID, + SVIFP_SHM_NUMBER_ATTACHED, + SVIFP_MQ_LAST_SEND_TIME, + SVIFP_MQ_LAST_RECEIVE_TIME, + SVIFP_MQ_LAST_CHANGE_TIME, + SVIFP_MQ_CURRENT_MESSAGES, + SVIFP_MQ_QUEUE_BYTES_MAX, + SVIFP_MQ_LAST_SEND_PID, + SVIFP_MQ_LAST_RECEIVE_PID +}; + +// Shorthand for lazy typists like me +#define IPC_CREX (IPC_CREAT | IPC_EXCL) + +#ifdef SYSV_IPC_DEBUG +#define DPRINTF(fmt, args...) fprintf(stderr, "+++ " fmt, ## args) +#else +#define DPRINTF(fmt, args...) +#endif + +/* ************************************************************************** +I have to do some guessing about types, mostly with regard to key_t. Since I +schlep these values back and forth between C and Python, I need to know +approximately what types these represent so that I can call the appropriate +Python function to convert them to Python values (e.g. PyInt_FromLong(), +PyFloat_FromDouble(), etc.) + +For most of these types the SUSv3 specification states that they're +integer-ish which means I can stuff them into a long and not worry. The macros +below safely convert each type and above each macro I document why my type +assumptions are safe. + +Unfortunately, key_t is an exception. The SUSv3 specification doesn't get more +detailed than saying it is "arithmetic", which includes signed and unsigned +short, int, long, long long, float and double (but thankfully excludes +pointers). I feel I can safely ignore long long since support for it is far +from ubiquitous. + +Ideally I would cast key_t to double and call PyWhatever_FromDouble() when +sending values to Python. Representing it as a double would ensure no data +loss regardless of whether it is typedef-ed as int, long, float, etc. + +In practice, this would be awkward. I have yet to see a platform where +key_t is not integer-ish, so most or all users of this library would be +surprised if it returned keys as floats. + +So I store keys as key_t types and cast them to long when Python forces +me to be specific. The two disadvantages to this are, (1) if any platforms +typedef key_t as a float or double, this code will break, and (2) KEY_MIN +and KEY_MAX don't represent the real min and max of keys. + +Point #1 is mitigated somewhat because if any OS typedefs key_t as float or +double, this code should complain loudly during compilation. + +Typedefs of key_t (for 32 bit systems): +OpenSolaris 2008.11 - int +OS X 10.5.8 - __int32_t (int) +Ubuntu 9.04 - int +Freebsd - l_int (int) +************************************************************************** */ + + +// key_t is guaranteed to be an arithmetic type. Some earlier versions +// of the standard didn't guarantee that it was arithmetic; the standard was +// changed to guarantee that it *is* arithmetic. +// ref: http://pubs.opengroup.org/onlinepubs/009696899/functions/xsh_chap02_12.html +// I assume it is a long; see comment above. +// Some functions return (key_t)-1, so I guess this has to be a signed type. +// ref: http://www.opengroup.org/austin/interps/doc.tpl?gdid=6226 +#if PY_MAJOR_VERSION > 2 + #define KEY_T_TO_PY(key) PyLong_FromLong(key) +#else + #define KEY_T_TO_PY(key) PyInt_FromLong(key) +#endif +// SUSv3 guarantees a uid_t to be an integer type. Some functions return +// (uid_t)-1, so I guess this has to be a signed type. +// ref: http://www.opengroup.org/onlinepubs/009695399/basedefs/sys/types.h.html +// ref: http://www.opengroup.org/onlinepubs/9699919799/functions/chown.html +#if PY_MAJOR_VERSION > 2 + #define UID_T_TO_PY(uid) PyLong_FromLong(uid) +#else + #define UID_T_TO_PY(uid) PyInt_FromLong(uid) +#endif + +// SUSv3 guarantees a gid_t to be an integer type. Some functions return +// (gid_t)-1, so I guess this has to be a signed type. +// ref: http://www.opengroup.org/onlinepubs/009695399/basedefs/sys/types.h.html +#if PY_MAJOR_VERSION > 2 + #define GID_T_TO_PY(gid) PyLong_FromLong(gid) +#else + #define GID_T_TO_PY(gid) PyInt_FromLong(gid) +#endif + +// I'm not sure what guarantees SUSv3 makes about a mode_t, but param 3 of +// shmget() is an int that contains flags in addition to the mode, so +// mode must be able to fit into an int. +// ref: http://www.opengroup.org/onlinepubs/009695399/functions/shmget.html +#if PY_MAJOR_VERSION > 2 + #define MODE_T_TO_PY(mode) PyLong_FromLong(mode) +#else + #define MODE_T_TO_PY(mode) PyInt_FromLong(mode) +#endif + +// I'm not sure what guarantees SUSv3 makes about a time_t, but the times +// I deal with here are all guaranteed to be after 1 Jan 1970 which means +// they'll always be positive numbers. A ulong sounds appropriate to me, +// and Python agrees in posixmodule.c. +#if PY_MAJOR_VERSION > 2 + #define TIME_T_TO_PY(time) PyLong_FromUnsignedLong(time) +#else + #define TIME_T_TO_PY(time) py_int_or_long_from_ulong(time) +#endif + +// C89 guarantees a size_t to be unsigned and fit into a ulong or smaller. +#if PY_MAJOR_VERSION > 2 + #define SIZE_T_TO_PY(size) PyLong_FromUnsignedLong(size) +#else + #define SIZE_T_TO_PY(size) py_int_or_long_from_ulong(size) +#endif + +// SUSv3 guarantees a pid_t to be a signed integer type. Some functions +// return (pid_t)-1 so I guess this has to be signed. +// ref: http://www.opengroup.org/onlinepubs/000095399/basedefs/sys/types.h.html#tag_13_67 +#if PY_MAJOR_VERSION > 2 + #define PID_T_TO_PY(pid) PyLong_FromLong(pid) +#else + #define PID_T_TO_PY(pid) PyInt_FromLong(pid) +#endif + +// The SUS guarantees a msglen_t to be an unsigned integer type. +// Ditto: msgqnum_t. +// ref: http://www.opengroup.org/onlinepubs/000095399/basedefs/sys/msg.h.html +#if PY_MAJOR_VERSION > 2 + #define MSGLEN_T_TO_PY(msglen) PyLong_FromUnsignedLong(msglen) + #define MSGQNUM_T_TO_PY(msgqnum) PyLong_FromUnsignedLong(msgqnum) +#else + #define MSGLEN_T_TO_PY(msglen) py_int_or_long_from_ulong(msglen) + #define MSGQNUM_T_TO_PY(msgqnum) py_int_or_long_from_ulong(msgqnum) +#endif + +/* Utility functions */ +key_t get_random_key(void); +int convert_key_param(PyObject *, void *); +#if PY_MAJOR_VERSION < 3 +PyObject *py_int_or_long_from_ulong(unsigned long); +#endif + +/* Custom Exceptions/Errors */ +extern PyObject *pBaseException; +extern PyObject *pInternalException; +extern PyObject *pPermissionsException; +extern PyObject *pExistentialException; +extern PyObject *pBusyException; +extern PyObject *pNotAttachedException; diff --git a/demo/ReadMe.txt b/demo/ReadMe.txt new file mode 100644 index 0000000..35d819b --- /dev/null +++ b/demo/ReadMe.txt @@ -0,0 +1,54 @@ +This demonstrates use of shared memory and semaphores via two applications +named after Mrs. Premise and Mrs. Conclusion of the Monty Python sketch. +http://www.youtube.com/watch?v=crIJvcWkVcs + +Like those two characters, these programs chat back and forth and the result +is a lot of nonsense. In this case, what the programs are saying isn't the +interesting part. What's interesting is how they're doing it. + +Mrs. Premise and Mrs. Conclusion (the programs, not the sketch characters) +communicate through IPC shared memory with a semaphore to control access +to that memory. + +Mrs. Premise starts things off by creating the shared memory and semaphore +and writing a random string (the current time) to the memory. She then sits +in a loop reading the memory. If it holds the same message she wrote, she +does nothing. If it is a new message, it must be from Mrs. Conclusion. + +Meanwhile, Mrs. Conclusion is doing exactly the same thing, except that she +assumes Mrs. Premise will write the first message. + +When either of these programs reads a new message, they write back an md5 +hash of that message. This serves two purposes. First, it ensures that +subsequent messages are very different so that if a message somehow gets +corrupted (say by being partially overwritten by the next message), it will +not escape notice. Second, it ensures that corruption can be detected if +it happens, because Mrs. Premise and Mrs. Conclusion can calculate what the +other's response to their message should be. + +Since they use a semaphore to control access to the shared memory, Mrs. +Premise and Mrs. Conclusion won't ever find their messages corrupted no +matter how many messages they exchange. You can experiment with this by +setting ITERATIONS in params.txt to a very large value. You can change +LIVE_DANGEROUSLY (also in params.txt) to a non-zero value to tell Mrs. +Premise and Mrs. Conclusion to run without using the semaphore. The shared +memory will probably get corrupted in fewer than 1000 iterations. + +To run the demo, start Mrs. Premise first in one window and then run +Mrs. Conclusion in another. + + + The Fancy Version + ================= + +If you want to get fancy, you can play with C versions of Mrs. Premise and +Mrs. Conclusion. The script make_all.sh will compile them for you. (Linux +users will need to edit the script and uncomment the line for the +Linux-specific linker option.) + +The resulting executables are called premise and conclusion and work exactly +the same as their Python counterparts. You can have the two C programs talk +to one another, or you can have premise.py talk to the C version of +conclusion...the possibilities are endless. (Actually, there are only four +possible combinations but "endless" sounds better.) + diff --git a/demo/SampleIpcConversation.png b/demo/SampleIpcConversation.png new file mode 100644 index 0000000..9d11e7a Binary files /dev/null and b/demo/SampleIpcConversation.png differ diff --git a/demo/cleanup.py b/demo/cleanup.py new file mode 100644 index 0000000..44c4ea6 --- /dev/null +++ b/demo/cleanup.py @@ -0,0 +1,29 @@ +# 3rd party modules +import sysv_ipc + +# Modules for this project +import utils + +params = utils.read_params() + + +try: + semaphore = sysv_ipc.Semaphore(params["KEY"], 0) +except: + semaphore = None + +if semaphore: + semaphore.remove() + +print ("The semaphore is cleaned up.") + + +try: + memory = sysv_ipc.SharedMemory(params["KEY"], 0) +except: + memory = None + +if memory: + memory.remove() + +print ("The shared memory is cleaned up.") diff --git a/demo/conclusion.c b/demo/conclusion.c new file mode 100644 index 0000000..7b10aef --- /dev/null +++ b/demo/conclusion.c @@ -0,0 +1,139 @@ +#include /* for system's IPC_xxx definitions */ +#include /* for shmget, shmat, shmdt, shmctl */ +#include /* for semget, semctl, semop */ + +#include +#include +#include +#include +#include + +#include "md5.h" +#include "utils.h" + +static const char MY_NAME[] = "Mrs. Conclusion"; + +// Set up a Mrs. Premise & Mrs. Conclusion conversation. + +int main() { + int sem_id = 0; + int shm_id = 0; + int rc; + char s[1024]; + int i; + int done; + char last_message_i_wrote[256]; + char md5ified_message[256]; + void *address = NULL; + struct param_struct params; + + say(MY_NAME, "Oooo 'ello, I'm Mrs. Conclusion!"); + + read_params(¶ms); + + // Mrs. Premise has already created the semaphore and shared memory. + // I just need to get handles to them. + sem_id = semget(params.key, 0, params.permissions); + + if (-1 == sem_id) { + sem_id = 0; + sprintf(s, "Getting a handle to the semaphore failed; errno is %d", errno); + say(MY_NAME, s); + } + else { + // get a handle to the shared memory + shm_id = shmget(params.key, params.size, params.permissions); + + if (shm_id == -1) { + shm_id = 0; + sprintf(s, "Couldn't get a handle to the shared memory; errno is %d", errno); + say(MY_NAME, s); + } + else { + sprintf(s, "Shared memory's id is %d", shm_id); + say(MY_NAME, s); + + // Attach the memory. + address = shmat(shm_id, NULL, 0); + + if ((void *)-1 == address) { + address = NULL; + sprintf(s, "Attaching the shared memory failed; errno is %d", errno); + say(MY_NAME, s); + } + else { + sprintf(s, "shared memory address = %p", address); + say(MY_NAME, s); + + i = 0; + done = 0; + last_message_i_wrote[0] = '\0'; + while (!done) { + sprintf(s, "iteration %d", i); + say(MY_NAME, s); + + // Wait for Mrs. Premise to free up the semaphore. + rc = acquire_semaphore(MY_NAME, sem_id, params.live_dangerously); + if (rc) + done = 1; + else { + while ( (!rc) && \ + (!strcmp((char *)address, last_message_i_wrote)) + ) { + // Nothing new; give Mrs. Premise another change to respond. + sprintf(s, "Read %zu characters '%s'", strlen((char *)address), (char *)address); + say(MY_NAME, s); + rc = release_semaphore(MY_NAME, sem_id, params.live_dangerously); + if (!rc) { + rc = acquire_semaphore(MY_NAME, sem_id, params.live_dangerously); + } + } + + md5ify(last_message_i_wrote, md5ified_message); + + // I always accept the first message (when i == 0) + if ( (i == 0) || (!strcmp(md5ified_message, (char *)address)) ) { + // All is well + i++; + + if (i == params.iterations) + done = 1; + + // MD5 the reply and write back to Mrs. Premise. + md5ify((char *)address, md5ified_message); + + // Write back to Mrs. Premise. + sprintf(s, "Writing %zu characters '%s'", strlen(md5ified_message), md5ified_message); + say(MY_NAME, s); + + strcpy((char *)address, md5ified_message); + + strcpy(last_message_i_wrote, md5ified_message); + } + else { + sprintf(s, "Shared memory corruption after %d iterations.", i); + say(MY_NAME, s); + sprintf(s, "Mismatch; rc = %d, new message is '%s', expected '%s'.", rc, (char *)address, md5ified_message); + say(MY_NAME, s); + done = 1; + } + } + + // Release the semaphore. + rc = release_semaphore(MY_NAME, sem_id, params.live_dangerously); + if (rc) + done = 1; + } + + if (-1 == shmdt(address)) { + sprintf(s, "Detaching the memory failed; errno is %d", errno); + say(MY_NAME, s); + } + address = NULL; + } + } + } + + + return 0; +} diff --git a/demo/conclusion.py b/demo/conclusion.py new file mode 100755 index 0000000..304f93e --- /dev/null +++ b/demo/conclusion.py @@ -0,0 +1,72 @@ +# Python modules +import os +import sys +PY_MAJOR_VERSION = sys.version_info[0] +# hashlib is only available in Python >= 2.5. I still want to support +# older Pythons so I import md5 if hashlib is not available. Fortunately +# md5 can masquerade as hashlib for my purposes. +try: + import hashlib +except ImportError: + import md5 as hashlib + +# 3rd party modules +import sysv_ipc + +# Utils for this demo +import utils +if PY_MAJOR_VERSION > 2: + import utils_for_3 as flex_utils +else: + import utils_for_2 as flex_utils + +utils.say("Oooo 'ello, I'm Mrs. Conclusion!") + +params = utils.read_params() + +semaphore = sysv_ipc.Semaphore(params["KEY"]) +memory = sysv_ipc.SharedMemory(params["KEY"]) + +utils.say("memory attached at %d" % memory.address) + +what_i_wrote = "" +s = "" + +for i in range(0, params["ITERATIONS"]): + utils.say("i = %d" % i) + if not params["LIVE_DANGEROUSLY"]: + # Wait for Mrs. Premise to free up the semaphore. + utils.say("acquiring the semaphore...") + semaphore.acquire() + + s = utils.read_from_memory(memory) + + while s == what_i_wrote: + if not params["LIVE_DANGEROUSLY"]: + # Release the semaphore... + utils.say("releasing the semaphore") + semaphore.release() + # ...and wait for it to become available again. + utils.say("acquiring for the semaphore...") + semaphore.acquire() + + s = utils.read_from_memory(memory) + + if what_i_wrote: + if PY_MAJOR_VERSION > 2: + what_i_wrote = what_i_wrote.encode() + try: + assert(s == hashlib.md5(what_i_wrote).hexdigest()) + except AssertionError: + flex_utils.raise_error(AssertionError, + "Shared memory corruption after %d iterations." % i) + + if PY_MAJOR_VERSION > 2: + s = s.encode() + what_i_wrote = hashlib.md5(s).hexdigest() + + utils.write_to_memory(memory, what_i_wrote) + + if not params["LIVE_DANGEROUSLY"]: + utils.say("releasing the semaphore") + semaphore.release() diff --git a/demo/make_all.sh b/demo/make_all.sh new file mode 100755 index 0000000..41e17c4 --- /dev/null +++ b/demo/make_all.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +gcc -Wall -c -o md5.o md5.c +gcc -Wall -c -o utils.o utils.c +gcc -Wall -L. md5.o utils.o -o premise premise.c +gcc -Wall -L. md5.o utils.o -o conclusion conclusion.c + diff --git a/demo/md5.c b/demo/md5.c new file mode 100644 index 0000000..c35d96c --- /dev/null +++ b/demo/md5.c @@ -0,0 +1,381 @@ +/* + Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.c is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order + either statically or dynamically; added missing #include + in library. + 2002-03-11 lpd Corrected argument list for main(), and added int return + type, in test program and T value program. + 2002-02-21 lpd Added missing #include in test program. + 2000-07-03 lpd Patched to eliminate warnings about "constant is + unsigned in ANSI C, signed in traditional"; made test program + self-checking. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5). + 1999-05-03 lpd Original version. + */ + +#include "md5.h" +#include + +#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */ +#ifdef ARCH_IS_BIG_ENDIAN +# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1) +#else +# define BYTE_ORDER 0 +#endif + +#define T_MASK ((md5_word_t)~0) +#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87) +#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9) +#define T3 0x242070db +#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111) +#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050) +#define T6 0x4787c62a +#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec) +#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe) +#define T9 0x698098d8 +#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850) +#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e) +#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841) +#define T13 0x6b901122 +#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c) +#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71) +#define T16 0x49b40821 +#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d) +#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf) +#define T19 0x265e5a51 +#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855) +#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2) +#define T22 0x02441453 +#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e) +#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437) +#define T25 0x21e1cde6 +#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829) +#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278) +#define T28 0x455a14ed +#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa) +#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07) +#define T31 0x676f02d9 +#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375) +#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd) +#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e) +#define T35 0x6d9d6122 +#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3) +#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb) +#define T38 0x4bdecfa9 +#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f) +#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f) +#define T41 0x289b7ec6 +#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805) +#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a) +#define T44 0x04881d05 +#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6) +#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a) +#define T47 0x1fa27cf8 +#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a) +#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb) +#define T50 0x432aff97 +#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58) +#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6) +#define T53 0x655b59c3 +#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d) +#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82) +#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e) +#define T57 0x6fa87e4f +#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f) +#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb) +#define T60 0x4e0811a1 +#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d) +#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca) +#define T63 0x2ad7d2bb +#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e) + + +static void +md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/) +{ + md5_word_t + a = pms->abcd[0], b = pms->abcd[1], + c = pms->abcd[2], d = pms->abcd[3]; + md5_word_t t; +#if BYTE_ORDER > 0 + /* Define storage only for big-endian CPUs. */ + md5_word_t X[16]; +#else + /* Define storage for little-endian or both types of CPUs. */ + md5_word_t xbuf[16]; + const md5_word_t *X; +#endif + + { +#if BYTE_ORDER == 0 + /* + * Determine dynamically whether this is a big-endian or + * little-endian machine, since we can use a more efficient + * algorithm on the latter. + */ + static const int w = 1; + + if (*((const md5_byte_t *)&w)) /* dynamic little-endian */ +#endif +#if BYTE_ORDER <= 0 /* little-endian */ + { + /* + * On little-endian machines, we can process properly aligned + * data without copying it. + */ + if (!((data - (const md5_byte_t *)0) & 3)) { + /* data are properly aligned */ + X = (const md5_word_t *)data; + } else { + /* not aligned */ + memcpy(xbuf, data, 64); + X = xbuf; + } + } +#endif +#if BYTE_ORDER == 0 + else /* dynamic big-endian */ +#endif +#if BYTE_ORDER >= 0 /* big-endian */ + { + /* + * On big-endian machines, we must arrange the bytes in the + * right order. + */ + const md5_byte_t *xp = data; + int i; + +# if BYTE_ORDER == 0 + X = xbuf; /* (dynamic only) */ +# else +# define xbuf X /* (static only) */ +# endif + for (i = 0; i < 16; ++i, xp += 4) + xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24); + } +#endif + } + +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + + /* Round 1. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */ +#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + F(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 7, T1); + SET(d, a, b, c, 1, 12, T2); + SET(c, d, a, b, 2, 17, T3); + SET(b, c, d, a, 3, 22, T4); + SET(a, b, c, d, 4, 7, T5); + SET(d, a, b, c, 5, 12, T6); + SET(c, d, a, b, 6, 17, T7); + SET(b, c, d, a, 7, 22, T8); + SET(a, b, c, d, 8, 7, T9); + SET(d, a, b, c, 9, 12, T10); + SET(c, d, a, b, 10, 17, T11); + SET(b, c, d, a, 11, 22, T12); + SET(a, b, c, d, 12, 7, T13); + SET(d, a, b, c, 13, 12, T14); + SET(c, d, a, b, 14, 17, T15); + SET(b, c, d, a, 15, 22, T16); +#undef SET + + /* Round 2. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */ +#define G(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + G(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 1, 5, T17); + SET(d, a, b, c, 6, 9, T18); + SET(c, d, a, b, 11, 14, T19); + SET(b, c, d, a, 0, 20, T20); + SET(a, b, c, d, 5, 5, T21); + SET(d, a, b, c, 10, 9, T22); + SET(c, d, a, b, 15, 14, T23); + SET(b, c, d, a, 4, 20, T24); + SET(a, b, c, d, 9, 5, T25); + SET(d, a, b, c, 14, 9, T26); + SET(c, d, a, b, 3, 14, T27); + SET(b, c, d, a, 8, 20, T28); + SET(a, b, c, d, 13, 5, T29); + SET(d, a, b, c, 2, 9, T30); + SET(c, d, a, b, 7, 14, T31); + SET(b, c, d, a, 12, 20, T32); +#undef SET + + /* Round 3. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */ +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + H(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 5, 4, T33); + SET(d, a, b, c, 8, 11, T34); + SET(c, d, a, b, 11, 16, T35); + SET(b, c, d, a, 14, 23, T36); + SET(a, b, c, d, 1, 4, T37); + SET(d, a, b, c, 4, 11, T38); + SET(c, d, a, b, 7, 16, T39); + SET(b, c, d, a, 10, 23, T40); + SET(a, b, c, d, 13, 4, T41); + SET(d, a, b, c, 0, 11, T42); + SET(c, d, a, b, 3, 16, T43); + SET(b, c, d, a, 6, 23, T44); + SET(a, b, c, d, 9, 4, T45); + SET(d, a, b, c, 12, 11, T46); + SET(c, d, a, b, 15, 16, T47); + SET(b, c, d, a, 2, 23, T48); +#undef SET + + /* Round 4. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */ +#define I(x, y, z) ((y) ^ ((x) | ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + I(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 6, T49); + SET(d, a, b, c, 7, 10, T50); + SET(c, d, a, b, 14, 15, T51); + SET(b, c, d, a, 5, 21, T52); + SET(a, b, c, d, 12, 6, T53); + SET(d, a, b, c, 3, 10, T54); + SET(c, d, a, b, 10, 15, T55); + SET(b, c, d, a, 1, 21, T56); + SET(a, b, c, d, 8, 6, T57); + SET(d, a, b, c, 15, 10, T58); + SET(c, d, a, b, 6, 15, T59); + SET(b, c, d, a, 13, 21, T60); + SET(a, b, c, d, 4, 6, T61); + SET(d, a, b, c, 11, 10, T62); + SET(c, d, a, b, 2, 15, T63); + SET(b, c, d, a, 9, 21, T64); +#undef SET + + /* Then perform the following additions. (That is increment each + of the four registers by the value it had before this block + was started.) */ + pms->abcd[0] += a; + pms->abcd[1] += b; + pms->abcd[2] += c; + pms->abcd[3] += d; +} + +void +md5_init(md5_state_t *pms) +{ + pms->count[0] = pms->count[1] = 0; + pms->abcd[0] = 0x67452301; + pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476; + pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301; + pms->abcd[3] = 0x10325476; +} + +void +md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes) +{ + const md5_byte_t *p = data; + int left = nbytes; + int offset = (pms->count[0] >> 3) & 63; + md5_word_t nbits = (md5_word_t)(nbytes << 3); + + if (nbytes <= 0) + return; + + /* Update the message length. */ + pms->count[1] += nbytes >> 29; + pms->count[0] += nbits; + if (pms->count[0] < nbits) + pms->count[1]++; + + /* Process an initial partial block. */ + if (offset) { + int copy = (offset + nbytes > 64 ? 64 - offset : nbytes); + + memcpy(pms->buf + offset, p, copy); + if (offset + copy < 64) + return; + p += copy; + left -= copy; + md5_process(pms, pms->buf); + } + + /* Process full blocks. */ + for (; left >= 64; p += 64, left -= 64) + md5_process(pms, p); + + /* Process a final partial block. */ + if (left) + memcpy(pms->buf, p, left); +} + +void +md5_finish(md5_state_t *pms, md5_byte_t digest[16]) +{ + static const md5_byte_t pad[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + md5_byte_t data[8]; + int i; + + /* Save the length before padding. */ + for (i = 0; i < 8; ++i) + data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3)); + /* Pad to 56 bytes mod 64. */ + md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1); + /* Append the length. */ + md5_append(pms, data, 8); + for (i = 0; i < 16; ++i) + digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3)); +} diff --git a/demo/md5.h b/demo/md5.h new file mode 100644 index 0000000..698c995 --- /dev/null +++ b/demo/md5.h @@ -0,0 +1,91 @@ +/* + Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.h is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Removed support for non-ANSI compilers; removed + references to Ghostscript; clarified derivation from RFC 1321; + now handles byte order either statically or dynamically. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5); + added conditionalization for C++ compilation from Martin + Purschke . + 1999-05-03 lpd Original version. + */ + +#ifndef md5_INCLUDED +# define md5_INCLUDED + +/* + * This package supports both compile-time and run-time determination of CPU + * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be + * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is + * defined as non-zero, the code will be compiled to run only on big-endian + * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to + * run on either big- or little-endian CPUs, but will run slightly less + * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined. + */ + +typedef unsigned char md5_byte_t; /* 8-bit byte */ +typedef unsigned int md5_word_t; /* 32-bit word */ + +/* Define the state of the MD5 Algorithm. */ +typedef struct md5_state_s { + md5_word_t count[2]; /* message length in bits, lsw first */ + md5_word_t abcd[4]; /* digest buffer */ + md5_byte_t buf[64]; /* accumulate block */ +} md5_state_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Initialize the algorithm. */ +void md5_init(md5_state_t *pms); + +/* Append a string to the message. */ +void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes); + +/* Finish the message and return the digest. */ +void md5_finish(md5_state_t *pms, md5_byte_t digest[16]); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif /* md5_INCLUDED */ diff --git a/demo/params.txt b/demo/params.txt new file mode 100644 index 0000000..eff48bd --- /dev/null +++ b/demo/params.txt @@ -0,0 +1,18 @@ +# These parameters control how Mrs. Premise and Mrs. Conclusion behave. + +# ITERATIONS is the number of times they'll talk to one another. +# LIVE_DANGEROUSLY is a Boolean (0 or 1); if set to 1 the programs +# won't use the semaphore to coordinate access to the shared +# memory. Corruption will likely result. +# KEY is the key to be used for the semaphore and shared memory. +# PERMISSIONS are in octal (note the leading 0). +# SHM_SIZE is the size of the shared memory segment in bytes. + +ITERATIONS=1000 +LIVE_DANGEROUSLY=0 +KEY=42 +PERMISSIONS=0600 +SHM_SIZE=4096 + + + diff --git a/demo/premise.c b/demo/premise.c new file mode 100644 index 0000000..11cbb4d --- /dev/null +++ b/demo/premise.c @@ -0,0 +1,211 @@ +#include /* for system's IPC_xxx definitions */ +#include /* for shmget, shmat, shmdt, shmctl */ +#include /* for semget, semctl, semop */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "md5.h" +#include "utils.h" + +const char MY_NAME[] = "Mrs. Premise"; + +// Set up a Mrs. Premise & Mrs. Conclusion conversation. + +void get_current_time(char *); + +int main() { + int rc; + char s[1024]; + char last_message_i_wrote[256]; + char md5ified_message[256]; + int i = 0; + int done = 0; + struct param_struct params; + int shm_id; + void *address = NULL; + int sem_id; + struct shmid_ds shm_info; + + say(MY_NAME, "Oooo 'ello, I'm Mrs. Premise!"); + + read_params(¶ms); + + // Create the shared memory + shm_id = shmget(params.key, params.size, IPC_CREAT | IPC_EXCL | params.permissions); + + if (shm_id == -1) { + shm_id = 0; + sprintf(s, "Creating the shared memory failed; errno is %d", errno); + say(MY_NAME, s); + } + else { + sprintf(s, "Shared memory's id is %d", shm_id); + say(MY_NAME, s); + + // Attach the memory. + address = shmat(shm_id, NULL, 0); + + if ((void *)-1 == address) { + address = NULL; + sprintf(s, "Attaching the shared memory failed; errno is %d", errno); + say(MY_NAME, s); + } + else { + sprintf(s, "shared memory address = %p", address); + say(MY_NAME, s); + } + } + + if (address) { + // Create the semaphore + sem_id = semget(params.key, 1, IPC_CREAT | IPC_EXCL | params.permissions); + + if (-1 == sem_id) { + sem_id = 0; + sprintf(s, "Creating the semaphore failed; errno is %d", errno); + say(MY_NAME, s); + } + else { + sprintf(s, "the semaphore id is %d", sem_id); + say(MY_NAME, s); + + // I seed the shared memory with a random string (the current time). + get_current_time(s); + + strcpy((char *)address, s); + strcpy(last_message_i_wrote, s); + + sprintf(s, "Wrote %zu characters: %s", strlen(last_message_i_wrote), last_message_i_wrote); + say(MY_NAME, s); + + i = 0; + while (!done) { + sprintf(s, "iteration %d", i); + say(MY_NAME, s); + + // Release the semaphore... + rc = release_semaphore(MY_NAME, sem_id, params.live_dangerously); + // ...and wait for it to become available again. In real code + // I might want to sleep briefly before calling .acquire() in + // order to politely give other processes an opportunity to grab + // the semaphore while it is free so as to avoid starvation. But + // this code is meant to be a stress test that maximizes the + // opportunity for shared memory corruption and politeness is + // not helpful in stress tests. + if (!rc) + rc = acquire_semaphore(MY_NAME, sem_id, params.live_dangerously); + + if (rc) + done = 1; + else { + // I keep checking the shared memory until something new has + // been written. + while ( (!rc) && \ + (!strcmp((char *)address, last_message_i_wrote)) + ) { + // Nothing new; give Mrs. Conclusion another change to respond. + sprintf(s, "Read %zu characters '%s'", strlen((char *)address), (char *)address); + say(MY_NAME, s); + rc = release_semaphore(MY_NAME, sem_id, params.live_dangerously); + if (!rc) { + rc = acquire_semaphore(MY_NAME, sem_id, params.live_dangerously); + } + } + + + if (rc) + done = 1; + else { + sprintf(s, "Read %zu characters '%s'", strlen((char *)address), (char *)address); + say(MY_NAME, s); + + // What I read must be the md5 of what I wrote or something's + // gone wrong. + md5ify(last_message_i_wrote, md5ified_message); + + if (strcmp(md5ified_message, (char *)address) == 0) { + // Yes, the message is OK + i++; + if (i == params.iterations) + done = 1; + + // MD5 the reply and write back to Mrs. Conclusion. + md5ify(md5ified_message, md5ified_message); + + sprintf(s, "Writing %zu characters '%s'", strlen(md5ified_message), md5ified_message); + say(MY_NAME, s); + + strcpy((char *)address, md5ified_message); + strcpy((char *)last_message_i_wrote, md5ified_message); + } + else { + sprintf(s, "Shared memory corruption after %d iterations.", i); + say(MY_NAME, s); + sprintf(s, "Mismatch; new message is '%s', expected '%s'.", (char *)address, md5ified_message); + say(MY_NAME, s); + done = 1; + } + } + } + } + + // Announce for one last time that the semaphore is free again so that + // Mrs. Conclusion can exit. + say(MY_NAME, "Final release of the semaphore followed by a 5 second pause"); + rc = release_semaphore(MY_NAME, sem_id, params.live_dangerously); + sleep(5); + // ...before beginning to wait until it is free again. + // Technically, this is bad practice. It's possible that on a + // heavily loaded machine, Mrs. Conclusion wouldn't get a chance + // to acquire the semaphore. There really ought to be a loop here + // that waits for some sort of goodbye message but for purposes of + // simplicity I'm skipping that. + + say(MY_NAME, "Final wait to acquire the semaphore"); + rc = acquire_semaphore(MY_NAME, sem_id, params.live_dangerously); + if (!rc) { + say(MY_NAME, "Destroying the shared memory."); + + if (-1 == shmdt(address)) { + sprintf(s, "Detaching the memory failed; errno is %d", errno); + say(MY_NAME, s); + } + address = NULL; + + + if (-1 == shmctl(shm_id, IPC_RMID, &shm_info)) { + sprintf(s, "Removing the memory failed; errno is %d", errno); + say(MY_NAME, s); + } + } + } + + say(MY_NAME, "Destroying the semaphore."); + // Clean up the semaphore + if (-1 == semctl(sem_id, 0, IPC_RMID)) { + sprintf(s, "Removing the semaphore failed; errno is %d", errno); + say(MY_NAME, s); + } + } + return 0; +} + + +void get_current_time(char *s) { + time_t the_time; + struct tm *the_localtime; + char *pAscTime; + + the_time = time(NULL); + the_localtime = localtime(&the_time); + pAscTime = asctime(the_localtime); + + strcpy(s, pAscTime); +} diff --git a/demo/premise.py b/demo/premise.py new file mode 100755 index 0000000..98dd30c --- /dev/null +++ b/demo/premise.py @@ -0,0 +1,106 @@ +# Python modules +import time +import os +import sys +PY_MAJOR_VERSION = sys.version_info[0] +# hashlib is only available in Python >= 2.5. I still want to support +# older Pythons so I import md5 if hashlib is not available. Fortunately +# md5 can masquerade as hashlib for my purposes. +try: + import hashlib +except ImportError: + import md5 as hashlib + +# 3rd party modules +import sysv_ipc + +# Utils for this demo +import utils +if PY_MAJOR_VERSION > 2: + import utils_for_3 as flex_utils +else: + import utils_for_2 as flex_utils + +utils.say("Oooo 'ello, I'm Mrs. Premise!") + +params = utils.read_params() + +# Create the semaphore & shared memory. I read somewhere that semaphores +# and shared memory have separate key spaces, so one can safely use the +# same key for each. This seems to be true in my experience. + +# For purposes of simplicity, this demo code makes no allowance for the +# failure of the semaphore or memory constructors. This is unrealistic +# because one can never predict whether or not a given key will be available, +# so your code must *always* be prepared for these functions to fail. + +semaphore = sysv_ipc.Semaphore(params["KEY"], sysv_ipc.IPC_CREX) +memory = sysv_ipc.SharedMemory(params["KEY"], sysv_ipc.IPC_CREX) + +# I seed the shared memory with a random value which is the current time. +what_i_wrote = time.asctime() +s = what_i_wrote + +utils.write_to_memory(memory, what_i_wrote) + +for i in range(0, params["ITERATIONS"]): + utils.say("iteration %d" % i) + if not params["LIVE_DANGEROUSLY"]: + # Releasing the semaphore... + utils.say("releasing the semaphore") + semaphore.release() + # ...and wait for it to become available again. In real code it'd be + # wise to sleep briefly before calling .acquire() in order to be + # polite and give other processes an opportunity to grab the semaphore + # while it is free and thereby avoid starvation. But this code is meant + # to be a stress test that maximizes the opportunity for shared memory + # corruption, and politeness has no place in that. + utils.say("acquiring the semaphore...") + semaphore.acquire() + + s = utils.read_from_memory(memory) + + # I keep checking the shared memory until something new has been written. + while s == what_i_wrote: + if not params["LIVE_DANGEROUSLY"]: + utils.say("releasing the semaphore") + semaphore.release() + utils.say("acquiring the semaphore...") + semaphore.acquire() + + # Once the call to .acquire() completes, I own the shared resource and + # I'm free to read from the memory. + s = utils.read_from_memory(memory) + + # What I read must be the md5 of what I wrote or something's gone wrong. + if PY_MAJOR_VERSION > 2: + what_i_wrote = what_i_wrote.encode() + + try: + assert(s == hashlib.md5(what_i_wrote).hexdigest()) + except AssertionError: + flex_utils.raise_error(AssertionError, + "Shared memory corruption after %d iterations." % i) + + # MD5 the reply and write back to Mrs. Conclusion. + if PY_MAJOR_VERSION > 2: + s = s.encode() + what_i_wrote = hashlib.md5(s).hexdigest() + utils.write_to_memory(memory, what_i_wrote) + + +# Announce for one last time that the semaphore is free again so that +# Mrs. Conclusion can exit. +if not params["LIVE_DANGEROUSLY"]: + utils.say("Final release of the semaphore followed by a 5 second pause") + semaphore.release() + time.sleep(5) + # ...before beginning to wait until it is free again. + utils.say("Final acquisition of the semaphore") + semaphore.acquire() + +utils.say("Destroying semaphore and shared memory") +# It'd be more natural to call memory.remove() and semaphore.remove() here, +# but I'll use the module-level functions instead to demonstrate their use. +sysv_ipc.remove_shared_memory(memory.id) +sysv_ipc.remove_semaphore(semaphore.id) diff --git a/demo/utils.c b/demo/utils.c new file mode 100644 index 0000000..30780c4 --- /dev/null +++ b/demo/utils.c @@ -0,0 +1,128 @@ +#include +#include +#include +#include +#include +#include +#include + + +#include /* for system's IPC_xxx definitions */ +#include /* for shmget, shmat, shmdt, shmctl */ +#include /* for semget, semctl, semop */ + +#include "utils.h" +#include "md5.h" + + +void md5ify(char *inString, char *outString) { + md5_state_t state; + md5_byte_t digest[16]; + int i; + + md5_init(&state); + md5_append(&state, (const md5_byte_t *)inString, strlen(inString)); + md5_finish(&state, digest); + + for (i = 0; i < 16; i++) + sprintf(&outString[i * 2], "%02x", digest[i]); +} + +void say(const char *pName, char *pMessage) { + time_t the_time; + struct tm *the_localtime; + char timestamp[256]; + + the_time = time(NULL); + + the_localtime = localtime(&the_time); + + strftime(timestamp, 255, "%H:%M:%S", the_localtime); + + printf("%s @ %s: %s\n", pName, timestamp, pMessage); + +} + + +int release_semaphore(const char *pName, int sem_id, int live_dangerously) { + int rc = 0; + struct sembuf op[1]; + char s[1024]; + + say(pName, "Releasing the semaphore."); + + if (!live_dangerously) { + op[0].sem_num = 0; + op[0].sem_op = 1; + op[0].sem_flg = 0; + + if (-1 == semop(sem_id, op, (size_t)1)) { + sprintf(s, "Releasing the semaphore failed; errno is %d\n", errno); + say(pName, s); + } + } + + return rc; +} + + +int acquire_semaphore(const char *pName, int sem_id, int live_dangerously) { + int rc = 0; + struct sembuf op[1]; + char s[1024]; + + say(pName, "Waiting to acquire the semaphore."); + + if (!live_dangerously) { + op[0].sem_num = 0; + op[0].sem_op = -1; + op[0].sem_flg = 0; + if (-1 == semop(sem_id, op, (size_t)1)) { + sprintf(s, "Acquiring the semaphore failed; errno is %d\n", errno); + say(pName, s); + } + } + + return rc; +} + + +void read_params(struct param_struct *params) { + char line[1024]; + char name[1024]; + int value = 0; + + FILE *fp; + + fp = fopen("params.txt", "r"); + + while (fgets(line, 1024, fp)) { + if (strlen(line) && ('#' == line[0])) + ; // comment in input, ignore + else { + sscanf(line, "%[ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghjiklmnopqrstuvwxyz]=%i\n", name, &value); + + // printf("name = %s, value = %d\n", name, value); + + if (!strcmp(name, "ITERATIONS")) + params->iterations = value; + if (!strcmp(name, "LIVE_DANGEROUSLY")) + params->live_dangerously = value; + if (!strcmp(name, "KEY")) + params->key = value; + if (!strcmp(name, "PERMISSIONS")) + params->permissions = value; + if (!strcmp(name, "SHM_SIZE")) + params->size = value; + + name[0] = '\0'; + value = 0; + } + } + + // printf("iterations = %d\n", params->iterations); + // printf("danger = %d\n", params->live_dangerously); + // printf("key = %d\n", params->key); + // printf("permissions = %o\n", params->permissions); + // printf("size = %d\n", params->size); +} diff --git a/demo/utils.h b/demo/utils.h new file mode 100644 index 0000000..9564d63 --- /dev/null +++ b/demo/utils.h @@ -0,0 +1,17 @@ +struct param_struct { + int iterations; + int live_dangerously; + int key; + int permissions; + int size; +}; + + +void md5ify(char *, char *); +void say(const char *, char *); +int acquire_semaphore(const char *, int, int); +int release_semaphore(const char *, int, int); +void read_params(struct param_struct *); + + + diff --git a/demo/utils.py b/demo/utils.py new file mode 100644 index 0000000..a1f4db2 --- /dev/null +++ b/demo/utils.py @@ -0,0 +1,64 @@ +import time +import sys + +PY_MAJOR_VERSION = sys.version_info[0] + +if PY_MAJOR_VERSION > 2: + NULL_CHAR = 0 +else: + NULL_CHAR = '\0' + +def say(s): + who = sys.argv[0] + if who.endswith(".py"): + who = who[:-3] + s = "%s@%1.6f: %s" % (who, time.time(), s) + print (s) + +def write_to_memory(memory, s): + say("writing %s " % s) + # Pad with NULLs in case I'm communicating with a C program. + #memory.write(s + (memory.size - len(s)) * '\0') + s += '\0' + if PY_MAJOR_VERSION > 2: + s = s.encode() + memory.write(s) + +def read_from_memory(memory): + s = memory.read() + if PY_MAJOR_VERSION > 2: + s = s.decode() + i = s.find('\0') + if i != -1: + s = s[:i] + say("read %s" % s) + + return s + +def read_params(): + params = { } + + f = open("params.txt", "r") + + for line in f: + line = line.strip() + if len(line): + if line.startswith('#'): + pass # comment in input, ignore + else: + name, value = line.split('=') + name = name.upper().strip() + + if name == "PERMISSIONS": + value = int(value, 8) + else: + value = int(value) + + #print "name = %s, value = %d" % (name, value) + + params[name] = value + + f.close() + + return params + diff --git a/demo/utils_for_2.py b/demo/utils_for_2.py new file mode 100644 index 0000000..c00f914 --- /dev/null +++ b/demo/utils_for_2.py @@ -0,0 +1,4 @@ + +def raise_error(error, message): + raise error, message + diff --git a/demo/utils_for_3.py b/demo/utils_for_3.py new file mode 100644 index 0000000..34155ae --- /dev/null +++ b/demo/utils_for_3.py @@ -0,0 +1,4 @@ + +def raise_error(error, message): + raise error(message) + diff --git a/demo2/ReadMe.txt b/demo2/ReadMe.txt new file mode 100644 index 0000000..871eba6 --- /dev/null +++ b/demo2/ReadMe.txt @@ -0,0 +1,41 @@ +This demonstrates use of message queues via two applications named after +Mrs. Premise and Mrs. Conclusion of the Monty Python sketch. +http://www.youtube.com/watch?v=crIJvcWkVcs + +Like those two characters, these programs chat back and forth and the result +is a lot of nonsense. In this case, what the programs are saying isn't the +interesting part. What's interesting is how they're doing it. + +Mrs. Premise and Mrs. Conclusion (the programs, not the sketch characters) +communicate with Sys V message queues. + +Mrs. Premise starts things off by creating the queue and sending a random +string (the current time) to it. She then sits in a loop receiving whatever +message is on the queue. If it is the same message she wrote, she sends it +back to the queue. If it is a new message, it must be from Mrs. Conclusion. + +Meanwhile, Mrs. Conclusion is doing exactly the same thing, except that she +assumes Mrs. Premise will write the first message. + +When either of these programs receives a new message, they send back an +md5 hash of that message. This serves two purposes. First, it ensures that +subsequent messages are very different so that if a message somehow gets +corrupted (say by being partially overwritten by the next message), it will +not escape notice. Second, it ensures that corruption can be detected if +it happens, because Mrs. Premise and Mrs. Conclusion can calculate what the +other's response to their message should be. + +Since message queues manage all of the concurrency issues transparently, +Mrs. Premise and Mrs. Conclusion won't ever find their messages corrupted +no matter how many messages they exchange. You can experiment with this by +setting ITERATIONS in params.txt to a very large value. + +These programs are not meant as a demonstration on how to make best use of a +message queue. In fact, they're very badly behaved because they poll the +queue as fast as possible -- they'll send your CPU usage right up to 100%. +Remember, they're trying as hard as they can to step one another so as to +expose any concurrency problems that might be present. + +Real code would want to sleep (or do something useful) in between calling +send() and receive(). + diff --git a/demo2/SampleIpcConversation.png b/demo2/SampleIpcConversation.png new file mode 100644 index 0000000..9d11e7a Binary files /dev/null and b/demo2/SampleIpcConversation.png differ diff --git a/demo2/cleanup.py b/demo2/cleanup.py new file mode 100755 index 0000000..3bc2aac --- /dev/null +++ b/demo2/cleanup.py @@ -0,0 +1,17 @@ +import sysv_ipc +import utils + +params = utils.read_params() + + +try: + mq = sysv_ipc.MessageQueue(params["KEY"]) + mq.remove() + s = "message queue %d removed" % params["KEY"] + print (s) +except: + print ("message queue doesn't exist") + + + +print ("\nAll clean!") \ No newline at end of file diff --git a/demo2/conclusion.py b/demo2/conclusion.py new file mode 100644 index 0000000..3e1701a --- /dev/null +++ b/demo2/conclusion.py @@ -0,0 +1,66 @@ +# Python modules +import time +import sys +PY_MAJOR_VERSION = sys.version_info[0] +# hashlib is only available in Python >= 2.5. I still want to support +# older Pythons so I import md5 if hashlib is not available. Fortunately +# md5 can masquerade as hashlib for my purposes. +try: + import hashlib +except ImportError: + import md5 as hashlib + +# 3rd party modules +import sysv_ipc + +# Utils for this demo +import utils +if PY_MAJOR_VERSION > 2: + import utils_for_3 as flex_utils +else: + import utils_for_2 as flex_utils + +params = utils.read_params() + +# Mrs. Premise has already created the message queue. I just need a handle +# to it. +mq = sysv_ipc.MessageQueue(params["KEY"]) + +what_i_sent = "" + +for i in range(0, params["ITERATIONS"]): + utils.say("iteration %d" % i) + + s, _ = mq.receive() + s = s.decode() + utils.say("Received %s" % s) + + while s == what_i_sent: + # Nothing new; give Mrs. Premise another chance to respond. + mq.send(s) + + s, _ = mq.receive() + s = s.decode() + utils.say("Received %s" % s) + + if what_i_sent: + if PY_MAJOR_VERSION > 2: + what_i_sent = what_i_sent.encode() + try: + assert(s == hashlib.md5(what_i_sent).hexdigest()) + except: + flex_utils.raise_error(AssertionError, + "Message corruption after %d iterations." % i) + #else: + # When what_i_sent is blank, this is the first message which + # I always accept without question. + + # MD5 the reply and write back to Mrs. Premise. + s = hashlib.md5(s.encode()).hexdigest() + utils.say("Sending %s" % s) + mq.send(s) + what_i_sent = s + + +utils.say("") +utils.say("%d iterations complete" % (i + 1)) diff --git a/demo2/params.txt b/demo2/params.txt new file mode 100644 index 0000000..e847cca --- /dev/null +++ b/demo2/params.txt @@ -0,0 +1,4 @@ +# These parameters control how Mrs. Premise and Mrs. Conclusion behave. + +ITERATIONS=1000 +KEY=42 diff --git a/demo2/premise.py b/demo2/premise.py new file mode 100644 index 0000000..942bba6 --- /dev/null +++ b/demo2/premise.py @@ -0,0 +1,71 @@ +# Python modules +import time +import sys +PY_MAJOR_VERSION = sys.version_info[0] +# hashlib is only available in Python >= 2.5. I still want to support +# older Pythons so I import md5 if hashlib is not available. Fortunately +# md5 can masquerade as hashlib for my purposes. +try: + import hashlib +except ImportError: + import md5 as hashlib + +# 3rd party modules +import sysv_ipc + +# Utils for this demo +import utils +if PY_MAJOR_VERSION > 2: + import utils_for_3 as flex_utils +else: + import utils_for_2 as flex_utils + +utils.say("Oooo 'ello, I'm Mrs. Premise!") + +params = utils.read_params() + +# Create the message queue. +mq = sysv_ipc.MessageQueue(params["KEY"], sysv_ipc.IPC_CREX) + +# The first message is a random string (the current time). +s = time.asctime() +utils.say("Sending %s" % s) +mq.send(s) +what_i_sent = s + +for i in range(0, params["ITERATIONS"]): + utils.say("iteration %d" % i) + + s, _ = mq.receive() + s = s.decode() + utils.say("Received %s" % s) + + # If the message is what I wrote, put it back on the queue. + while s == what_i_sent: + # Nothing new; give Mrs. Conclusion another chance to respond. + mq.send(s) + + s, _ = mq.receive() + s = s.decode() + utils.say("Received %s" % s) + + # What I read must be the md5 of what I wrote or something's + # gone wrong. + if PY_MAJOR_VERSION > 2: + what_i_sent = what_i_sent.encode() + try: + assert(s == hashlib.md5(what_i_sent).hexdigest()) + except AssertionError: + flex_utils.raise_error(AssertionError, "Message corruption after %d iterations." % i) + + # MD5 the reply and write back to Mrs. Conclusion. + s = hashlib.md5(s.encode()).hexdigest() + utils.say("Sending %s" % s) + mq.send(s) + what_i_sent = s + +utils.say("") +utils.say("%d iterations complete" % (i + 1)) + +utils.say("Destroying the message queue.") +mq.remove() diff --git a/demo2/utils.py b/demo2/utils.py new file mode 100644 index 0000000..0c66bee --- /dev/null +++ b/demo2/utils.py @@ -0,0 +1,41 @@ +import time +import sys + +def say(s): + who = sys.argv[0] + if who.endswith(".py"): + who = who[:-3] + s = "%s@%1.6f: %s" % (who, time.time(), s) + print (s) + + +def read_params(): + params = { } + + f = open("params.txt", "r") + + for line in f: + line = line.strip() + if len(line): + if line.startswith('#'): + pass # comment in input, ignore + else: + name, value = line.split('=') + name = name.upper().strip() + + if name == "PERMISSIONS": + value = int(value, 8) + elif "NAME" in name: + # This is a string; leave it alone. + pass + else: + value = int(value) + + #print "name = %s, value = %d" % (name, value) + + params[name] = value + + f.close() + + return params + diff --git a/demo2/utils_for_2.py b/demo2/utils_for_2.py new file mode 100644 index 0000000..c00f914 --- /dev/null +++ b/demo2/utils_for_2.py @@ -0,0 +1,4 @@ + +def raise_error(error, message): + raise error, message + diff --git a/demo2/utils_for_3.py b/demo2/utils_for_3.py new file mode 100644 index 0000000..34155ae --- /dev/null +++ b/demo2/utils_for_3.py @@ -0,0 +1,4 @@ + +def raise_error(error, message): + raise error(message) + diff --git a/demo4/ReadMe.txt b/demo4/ReadMe.txt new file mode 100644 index 0000000..25f73e9 --- /dev/null +++ b/demo4/ReadMe.txt @@ -0,0 +1,15 @@ +This demonstrates the use of a semaphore with a Python context manager (a +'with' statement). + +To run the demo, simply run `python parent.py`. It launches child.py. + +The programs parent.py and child.py share a semaphore; the latter acquires the +semaphore via a context manager. The child process deliberately kills itself via +an error about half the time (randomly) in order to demonstrate that the +context manager releases the semaphore regardless of whether or not the context +block is exited gracefully. + +Once the child releases the semaphore, the parent destroys it. + +The whole thing happens in less than 10 seconds. + diff --git a/demo4/child.py b/demo4/child.py new file mode 100644 index 0000000..f635769 --- /dev/null +++ b/demo4/child.py @@ -0,0 +1,23 @@ +import sysv_ipc +import time +import sys +import random + +# The parent passes the semaphore's name to me. +key = sys.argv[1] + +sem = sysv_ipc.Semaphore(int(key)) + +print('Child: waiting to aquire semaphore ' + key) + +with sem: + print('Child: semaphore {} aquired; holding for 3 seconds.'.format(sem.key)) + + # Flip a coin to determine whether or not to bail out of the context. + if random.randint(0, 1): + print("Child: raising ValueError to demonstrate unplanned context exit") + raise ValueError + + time.sleep(3) + + print('Child: gracefully exiting context (releasing the semaphore).') diff --git a/demo4/parent.py b/demo4/parent.py new file mode 100644 index 0000000..79f988c --- /dev/null +++ b/demo4/parent.py @@ -0,0 +1,46 @@ +import subprocess +import sysv_ipc +import time +import os + +sem = sysv_ipc.Semaphore(None, sysv_ipc.IPC_CREX, initial_value = 1) +print("Parent: created semaphore {}.".format(sem.key)) + +sem.acquire() + +# Spawn a child that will wait on this semaphore. +path, _ = os.path.split(__file__) +print("Parent: spawning child process...") +subprocess.Popen(["python", os.path.join(path, 'child.py'), str(sem.key)]) + +for i in range(3, 0, -1): + print("Parent: child process will acquire the semaphore in {} seconds...".format(i)) + time.sleep(1) + +sem.release() + +# Sleep for a second to give the child a chance to acquire the semaphore. +# This technique is a little sloppy because technically the child could still +# starve, but it's certainly sufficient for this demo. +time.sleep(1) + +# Wait for the child to release the semaphore. +print("Parent: waiting for the child to release the semaphore.") +sem.acquire() + +# Clean up. +print("Parent: destroying the semaphore.") +sem.release() +sem.remove() + +msg = """ +By the time you're done reading this, the parent will have exited and so the +operating system will have destroyed the semaphore. You can prove that the +semaphore is gone by running this command and observing that it raises +sysv_ipc.ExistentialError -- + + python -c "import sysv_ipc; sysv_ipc.Semaphore({})" + +""".format(sem.key) + +print(msg) diff --git a/ftok_experiment.py b/ftok_experiment.py new file mode 100644 index 0000000..95fcd21 --- /dev/null +++ b/ftok_experiment.py @@ -0,0 +1,50 @@ +"""This is an experiment to see how often ftok() returns duplicate keys for +different filenames. +""" + + +import sys +import os +import sysv_ipc + +if len(sys.argv) == 2: + start_path = sys.argv[1] +else: + msg = "Start path? [Default = your home directory] " + start_path = raw_input(msg) + if not start_path: + start_path = "~" + +# Expand paths that start with a tilde and then absolutize. +start_path = os.path.expanduser(start_path) +start_path = os.path.abspath(start_path) + +# For every filename in the tree, generate a key and associate the filename +# with that key via a dictionary. +d = { } +nfilenames = 0 +for path, dirnames, filenames in os.walk(start_path): + for filename in filenames: + # Fully qualify the path + filename = os.path.join(path, filename) + + nfilenames += 1 + + #print "Processing %s..." % filename + + key = sysv_ipc.ftok(filename, 42, True) + + if key not in d: + d[key] = [ ] + + d[key].append(filename) + +# Print statistics, including files with non-unique keys. +ndups = 0 +for key in d: + if len(d[key]) > 1: + ndups += len(d[key]) + print key, d[key] + +print "Out of {0} unique filenames, I found {1} duplicate keys.".format(nfilenames, ndups) + diff --git a/memory.c b/memory.c new file mode 100644 index 0000000..5fc5102 --- /dev/null +++ b/memory.c @@ -0,0 +1,834 @@ +#include "Python.h" +#include "structmember.h" + +#include "common.h" +#include "memory.h" + + +/****************** Internal use only **********************/ +PyObject * +shm_str(SharedMemory *self) { +#if PY_MAJOR_VERSION > 2 + return PyUnicode_FromFormat("Key=%ld, id=%d", (long)self->key, self->id); +#else + return PyString_FromFormat("Key=%ld, id=%d", (long)self->key, self->id); +#endif +} + +PyObject * +shm_repr(SharedMemory *self) { +#if PY_MAJOR_VERSION > 2 + return PyUnicode_FromFormat("sysv_ipc.SharedMemory(%ld)", (long)self->key); +#else + return PyString_FromFormat("sysv_ipc.SharedMemory(%ld)", (long)self->key); +#endif +} + +PyObject * +shm_attach(SharedMemory *self, int shmat_flags) { + DPRINTF("attaching memory @ address %p with id %d using flags 0x%x\n", + self->address, self->id, shmat_flags); + + self->address = shmat(self->id, self->address, shmat_flags); + + if ( (void *)-1 == self->address) { + self->address = NULL; + switch (errno) { + case EACCES: + PyErr_SetString(pPermissionsException, "No permission to attach"); + break; + + case ENOMEM: + PyErr_SetString(PyExc_MemoryError, "Not enough memory"); + break; + + case EINVAL: + PyErr_SetString(PyExc_ValueError, "Invalid id or address"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + + goto error_return; + } + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + +PyObject * +shm_remove(int shared_memory_id) { + struct shmid_ds shm_info; + + DPRINTF("removing shm with id %d\n", shared_memory_id); + if (-1 == shmctl(shared_memory_id, IPC_RMID, &shm_info)) { + switch (errno) { + case EIDRM: + case EINVAL: + PyErr_Format(pExistentialException, + "No shared memory with id %d exists", + shared_memory_id); + break; + + case EPERM: + PyErr_SetString(pPermissionsException, + "You do not have permission to remove the shared memory"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + +static PyObject * +shm_get_value(int shared_memory_id, enum GET_SET_IDENTIFIERS field) { + struct shmid_ds shm_info; + PyObject *py_value = NULL; + + DPRINTF("Calling shmctl(...IPC_STAT...), field = %d\n", field); + if (-1 == shmctl(shared_memory_id, IPC_STAT, &shm_info)) { + switch (errno) { + case EIDRM: + case EINVAL: + PyErr_Format(pExistentialException, + "No shared memory with id %d exists", + shared_memory_id); + break; + + case EACCES: + PyErr_SetString(pPermissionsException, + "You do not have permission to read the shared memory attribute"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + + goto error_return; + } + + switch (field) { + case SVIFP_SHM_SIZE: + py_value = SIZE_T_TO_PY(shm_info.shm_segsz); + break; + + case SVIFP_SHM_LAST_ATTACH_TIME: + py_value = TIME_T_TO_PY(shm_info.shm_atime); + break; + + case SVIFP_SHM_LAST_DETACH_TIME: + py_value = TIME_T_TO_PY(shm_info.shm_dtime); + break; + + case SVIFP_SHM_LAST_CHANGE_TIME: + py_value = TIME_T_TO_PY(shm_info.shm_ctime); + break; + + case SVIFP_SHM_CREATOR_PID: + py_value = PID_T_TO_PY(shm_info.shm_cpid); + break; + + case SVIFP_SHM_LAST_AT_DT_PID: + py_value = PID_T_TO_PY(shm_info.shm_lpid); + break; + + case SVIFP_SHM_NUMBER_ATTACHED: + // shm_nattch is unsigned + // ref: http://www.opengroup.org/onlinepubs/007908799/xsh/sysshm.h.html +#if PY_MAJOR_VERSION > 2 + py_value = PyLong_FromUnsignedLong(shm_info.shm_nattch); +#else + py_value = py_int_or_long_from_ulong(shm_info.shm_nattch); +#endif + break; + + case SVIFP_IPC_PERM_UID: + py_value = UID_T_TO_PY(shm_info.shm_perm.uid); + break; + + case SVIFP_IPC_PERM_GID: + py_value = GID_T_TO_PY(shm_info.shm_perm.gid); + break; + + case SVIFP_IPC_PERM_CUID: + py_value = UID_T_TO_PY(shm_info.shm_perm.cuid); + break; + + case SVIFP_IPC_PERM_CGID: + py_value = GID_T_TO_PY(shm_info.shm_perm.cgid); + break; + + case SVIFP_IPC_PERM_MODE: + py_value = MODE_T_TO_PY(shm_info.shm_perm.mode); + break; + + default: + PyErr_Format(pInternalException, "Bad field %d passed to shm_get_value", field); + goto error_return; + break; + } + + return py_value; + + error_return: + return NULL; +} + + +static int +shm_set_ipc_perm_value(int id, enum GET_SET_IDENTIFIERS field, union ipc_perm_value value) { + struct shmid_ds shm_info; + + if (-1 == shmctl(id, IPC_STAT, &shm_info)) { + switch (errno) { + case EIDRM: + case EINVAL: + PyErr_Format(pExistentialException, + "No shared memory with id %d exists", id); + break; + + case EACCES: + PyErr_SetString(pPermissionsException, + "You do not have permission to read the shared memory attribute"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + switch (field) { + case SVIFP_IPC_PERM_UID: + shm_info.shm_perm.uid = value.uid; + break; + + case SVIFP_IPC_PERM_GID: + shm_info.shm_perm.gid = value.gid; + break; + + case SVIFP_IPC_PERM_MODE: + shm_info.shm_perm.mode = value.mode; + break; + + default: + PyErr_Format(pInternalException, + "Bad field %d passed to shm_set_ipc_perm_value", + field); + goto error_return; + break; + } + + if (-1 == shmctl(id, IPC_SET, &shm_info)) { + switch (errno) { + case EIDRM: + case EINVAL: + PyErr_Format(pExistentialException, + "No shared memory with id %d exists", id); + break; + + case EPERM: + PyErr_SetString(pPermissionsException, + "You do not have permission to change the shared memory's attributes"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + + goto error_return; + } + + return 0; + + error_return: + return -1; +} + + + +/****************** Class methods **********************/ + + + +void +SharedMemory_dealloc(SharedMemory *self) { + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject * +SharedMemory_new(PyTypeObject *type, PyObject *args, PyObject *kwlist) { + SharedMemory *self; + + self = (SharedMemory *)type->tp_alloc(type, 0); + + if (NULL != self) { + self->key = (key_t)-1; + self->id = 0; + self->address = NULL; + } + + return (PyObject *)self; +} + + +int +SharedMemory_init(SharedMemory *self, PyObject *args, PyObject *keywords) { + NoneableKey key; + int mode = 0600; + unsigned long size = 0; + int shmget_flags = 0; + int shmat_flags = 0; + char init_character = ' '; + char *keyword_list[ ] = {"key", "flags", "mode", "size", "init_character", NULL}; + PyObject *py_size = NULL; + + DPRINTF("Inside SharedMemory_init()\n"); + + if (!PyArg_ParseTupleAndKeywords(args, keywords, "O&|iikc", keyword_list, + &convert_key_param, &key, + &shmget_flags, &mode, &size, + &init_character)) + goto error_return; + + mode &= 0777; + shmget_flags &= ~0777; + + DPRINTF("key is none = %d, key value = %ld\n", key.is_none, (long)key.value); + + if ( !(shmget_flags & IPC_CREAT) && (shmget_flags & IPC_EXCL) ) { + PyErr_SetString(PyExc_ValueError, + "IPC_EXCL must be combined with IPC_CREAT"); + goto error_return; + } + + if (key.is_none && ((shmget_flags & IPC_EXCL) != IPC_EXCL)) { + PyErr_SetString(PyExc_ValueError, + "Key can only be None if IPC_EXCL is set"); + goto error_return; + } + + // When creating a new segment, the default size is PAGE_SIZE. + if (((shmget_flags & IPC_CREX) == IPC_CREX) && (!size)) + size = PAGE_SIZE; + + if (key.is_none) { + // (key == None) ==> generate a key for the caller + do { + errno = 0; + self->key = get_random_key(); + + DPRINTF("Calling shmget, key=%ld, size=%lu, mode=%o, flags=0x%x\n", + (long)self->key, size, mode, shmget_flags); + self->id = shmget(self->key, size, mode | shmget_flags); + } while ( (-1 == self->id) && (EEXIST == errno) ); + } + else { + // (key != None) ==> use key supplied by the caller + self->key = key.value; + + DPRINTF("Calling shmget, key=%ld, size=%lu, mode=%o, flags=0x%x\n", + (long)self->key, size, mode, shmget_flags); + self->id = shmget(self->key, size, mode | shmget_flags); + } + + DPRINTF("id == %d\n", self->id); + + if (self->id == -1) { + switch (errno) { + case EACCES: + PyErr_Format(pPermissionsException, + "Permission %o cannot be granted on the existing segment", + mode); + break; + + case EEXIST: + PyErr_Format(pExistentialException, + "Shared memory with the key %ld already exists", + (long)self->key); + break; + + case ENOENT: + PyErr_Format(pExistentialException, + "No shared memory exists with the key %ld", (long)self->key); + break; + + case EINVAL: + PyErr_SetString(PyExc_ValueError, "The size is invalid"); + break; + + case ENOMEM: + PyErr_SetString(PyExc_MemoryError, "Not enough memory"); + break; + + case ENOSPC: + PyErr_SetString(PyExc_OSError, + "Not enough shared memory identifiers available (ENOSPC)"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + // Attach the memory. If no write permissions requested, attach read-only. + shmat_flags = (mode & 0200) ? 0 : SHM_RDONLY; + if (NULL == shm_attach(self, shmat_flags)) { + // Bad news, something went wrong. + goto error_return; + } + + if ( ((shmget_flags & IPC_CREX) == IPC_CREX) && (!(shmat_flags & SHM_RDONLY)) ) { + // Initialize the memory. + + py_size = shm_get_value(self->id, SVIFP_SHM_SIZE); + + if (!py_size) + goto error_return; + else { +#if PY_MAJOR_VERSION > 2 + size = PyLong_AsUnsignedLongMask(py_size); +#else + size = PyInt_AsUnsignedLongMask(py_size); +#endif + + DPRINTF("memsetting address %p to %lu bytes of ASCII 0x%x (%c)\n", \ + self->address, size, (int)init_character, init_character); + memset(self->address, init_character, size); + } + + Py_DECREF(py_size); + } + + return 0; + + error_return: + return -1; +} + + +PyObject * +SharedMemory_attach(SharedMemory *self, PyObject *args) { + PyObject *py_address = NULL; + void *address = NULL; + int flags = 0; + + if (!PyArg_ParseTuple(args, "|Oi", &py_address, &flags)) + goto error_return; + + if ((!py_address) || (py_address == Py_None)) + address = NULL; + else { + if (PyLong_Check(py_address)) + address = PyLong_AsVoidPtr(py_address); + else { + PyErr_SetString(PyExc_TypeError, "address must be a long"); + goto error_return; + } + } + + self->address = shmat(self->id, address, flags); + + if ((void *)-1 == self->address) { + self->address = NULL; + switch (errno) { + case EACCES: + PyErr_SetString(pPermissionsException, "No permission to attach"); + break; + + case EINVAL: + PyErr_SetString(PyExc_ValueError, "Invalid address or flags"); + break; + + case ENOMEM: + PyErr_SetString(PyExc_MemoryError, "Not enough memory"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + +PyObject * +SharedMemory_detach(SharedMemory *self) { + if (-1 == shmdt(self->address)) { + self->address = NULL; + switch (errno) { + case EINVAL: + PyErr_SetNone(pNotAttachedException); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + self->address = NULL; + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + +PyObject * +SharedMemory_read(SharedMemory *self, PyObject *args, PyObject *keywords) { + /* Tricky business here. A memory segment's size is a size_t which is + ulong or smaller. However, the largest string that Python can + construct is of ssize_t which is long or smaller. Therefore, the + size and offset variables must be ulongs while the byte_count + must be a long (and must not exceed LONG_MAX). + Mind your math! + */ + long byte_count = 0; + unsigned long offset = 0; + unsigned long size; + PyObject *py_size; + char *keyword_list[ ] = {"byte_count", "offset", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, keywords, "|lk", keyword_list, + &byte_count, &offset)) + goto error_return; + + if (self->address == NULL) { + PyErr_SetString(pNotAttachedException, + "Read attempt on unattached memory segment"); + goto error_return; + } + + if ( (py_size = shm_get_value(self->id, SVIFP_SHM_SIZE)) ) { +#if PY_MAJOR_VERSION > 2 + size = PyLong_AsUnsignedLongMask(py_size); +#else + size = PyInt_AsUnsignedLongMask(py_size); +#endif + Py_DECREF(py_size); + } + else + goto error_return; + + DPRINTF("offset = %lu, byte_count = %ld, size = %lu\n", + offset, byte_count, size); + + if (offset >= size) { + PyErr_SetString(PyExc_ValueError, "The offset must be less than the segment size"); + goto error_return; + } + + if (byte_count < 0) { + PyErr_SetString(PyExc_ValueError, "The byte_count cannot be negative"); + goto error_return; + } + + /* If the caller didn't specify a byte count or specified one that would + read past the end of the segment, return everything from the offset to + the end of the segment. + Be careful here not to express the second if condition w/addition, e.g. + (byte_count + offset > size) + It might be more intuitive but since byte_count is a long and offset + is a ulong, their sum could cause an arithmetic overflow. */ + if ((!byte_count) || ((unsigned long)byte_count > size - offset)) { + // byte_count needs to be calculated + if (size - offset <= (unsigned long)PY_STRING_LENGTH_MAX) + byte_count = size - offset; + else { + // Caller is asking for more bytes than I can stuff into + // a Python string. + PyErr_Format(PyExc_ValueError, + "The byte_count cannot exceed Python's max string length %ld", + (long)PY_STRING_LENGTH_MAX); + goto error_return; + } + } + + +#if PY_MAJOR_VERSION > 2 + return PyBytes_FromStringAndSize(self->address + offset, byte_count); +#else + return PyString_FromStringAndSize(self->address + offset, byte_count); +#endif + + error_return: + return NULL; +} + + +PyObject * +SharedMemory_write(SharedMemory *self, PyObject *args, PyObject *kw) { + /* See comments for read() regarding "size issues". Note that here + Python provides the byte_count so it can't be negative. + + In Python >= 2.5, the Python argument specifier 's#' expects a + py_ssize_t for its second parameter. A long is long enough. It might + be too big, though, on platforms where a long is larger than + py_ssize_t. Therefore I *must* initialize it to 0 so that whatever + Python doesn't write to is zeroed out. + */ + unsigned long offset = 0; + unsigned long size; + PyObject *py_size; + char *keyword_list[ ] = {"s", "offset", NULL}; +#if PY_MAJOR_VERSION > 2 + static char args_format[] = "s*|l"; + Py_buffer data; +#else + static char args_format[] = "s#|l"; + typedef struct { + const char *buf; + long len; + } MyBuffer; + MyBuffer data; + data.len = 0; +#endif + + if (!PyArg_ParseTupleAndKeywords(args, kw, args_format, keyword_list, +#if PY_MAJOR_VERSION > 2 + &data, +#else + &(data.buf), &(data.len), +#endif + &offset)) + goto error_return; + + if (self->address == NULL) { + PyErr_SetString(pNotAttachedException, "Write attempt on unattached memory segment"); + goto error_return; + } + + if ( (py_size = shm_get_value(self->id, SVIFP_SHM_SIZE)) ) { +#if PY_MAJOR_VERSION > 2 + size = PyLong_AsUnsignedLongMask(py_size); +#else + size = PyInt_AsUnsignedLongMask(py_size); +#endif + Py_DECREF(py_size); + } + else + goto error_return; + + DPRINTF("write size check; size=%lu, offset=%lu, dat.len=%ld\n", + size, offset, data.len); + + if ((unsigned long)data.len > size - offset) { + PyErr_SetString(PyExc_ValueError, "Attempt to write past end of memory segment"); + goto error_return; + } + + memcpy((self->address + offset), data.buf, data.len); + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + +PyObject * +SharedMemory_remove(SharedMemory *self) { + return shm_remove(self->id); +} + + +PyObject * +shm_get_key(SharedMemory *self) { + return KEY_T_TO_PY(self->key); +} + +PyObject * +shm_get_size(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_SHM_SIZE); +} + +PyObject * +shm_get_address(SharedMemory *self) { + return PyLong_FromVoidPtr(self->address); +} + +PyObject * +shm_get_attached(SharedMemory *self) { + if (self->address) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + +PyObject * +shm_get_last_attach_time(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_SHM_LAST_ATTACH_TIME); +} + +PyObject * +shm_get_last_detach_time(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_SHM_LAST_DETACH_TIME); +} + +PyObject * +shm_get_last_change_time(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_SHM_LAST_CHANGE_TIME); +} + +PyObject * +shm_get_creator_pid(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_SHM_CREATOR_PID); +} + +PyObject * +shm_get_last_pid(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_SHM_LAST_AT_DT_PID); +} + +PyObject * +shm_get_number_attached(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_SHM_NUMBER_ATTACHED); +} + +PyObject * +shm_get_uid(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_IPC_PERM_UID); +} + +PyObject * +shm_get_cuid(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_IPC_PERM_CUID); +} + +PyObject * +shm_get_cgid(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_IPC_PERM_CGID); +} + +PyObject * +shm_get_mode(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_IPC_PERM_MODE); +} + +int +shm_set_uid(SharedMemory *self, PyObject *py_value) { + union ipc_perm_value new_value; + +#if PY_MAJOR_VERSION > 2 + if (!PyLong_Check(py_value)) +#else + if (!PyInt_Check(py_value)) +#endif + { + PyErr_SetString(PyExc_TypeError, "Attribute 'uid' must be an integer"); + goto error_return; + } + +#if PY_MAJOR_VERSION > 2 + new_value.uid = PyLong_AsLong(py_value); +#else + new_value.uid = PyInt_AsLong(py_value); +#endif + + if (((uid_t)-1 == new_value.uid) && PyErr_Occurred()) { + // no idea what could have gone wrong -- punt it up to the caller + goto error_return; + } + + return shm_set_ipc_perm_value(self->id, SVIFP_IPC_PERM_UID, new_value); + + error_return: + return -1; +} + + +PyObject * +shm_get_gid(SharedMemory *self) { + return shm_get_value(self->id, SVIFP_IPC_PERM_GID); +} + +int +shm_set_gid(SharedMemory *self, PyObject *py_value) { + union ipc_perm_value new_value; + +#if PY_MAJOR_VERSION > 2 + if (!PyLong_Check(py_value)) +#else + if (!PyInt_Check(py_value)) +#endif + { + PyErr_Format(PyExc_TypeError, "attribute 'gid' must be an integer"); + goto error_return; + } + +#if PY_MAJOR_VERSION > 2 + new_value.gid = PyLong_AsLong(py_value); +#else + new_value.gid = PyInt_AsLong(py_value); +#endif + + if (((gid_t)-1 == new_value.gid) && PyErr_Occurred()) { + // no idea what could have gone wrong -- punt it up to the caller + goto error_return; + } + + return shm_set_ipc_perm_value(self->id, SVIFP_IPC_PERM_GID, new_value); + + error_return: + return -1; +} + +int +shm_set_mode(SharedMemory *self, PyObject *py_value) { + union ipc_perm_value new_value; + +#if PY_MAJOR_VERSION > 2 + if (!PyLong_Check(py_value)) +#else + if (!PyInt_Check(py_value)) +#endif + { + PyErr_Format(PyExc_TypeError, "attribute 'mode' must be an integer"); + goto error_return; + } + +#if PY_MAJOR_VERSION > 2 + new_value.mode = PyLong_AsLong(py_value); +#else + new_value.mode = PyInt_AsLong(py_value); +#endif + + if (((mode_t)-1 == new_value.mode) && PyErr_Occurred()) { + // no idea what could have gone wrong -- punt it up to the caller + goto error_return; + } + + return shm_set_ipc_perm_value(self->id, SVIFP_IPC_PERM_MODE, new_value); + + error_return: + return -1; +} + diff --git a/memory.h b/memory.h new file mode 100644 index 0000000..d6c3cfc --- /dev/null +++ b/memory.h @@ -0,0 +1,57 @@ +typedef struct { + PyObject_HEAD + key_t key; + int id; + void *address; +} SharedMemory; + +/* Union for passing values to shm_set_ipc_perm_value() */ +union ipc_perm_value { + uid_t uid; + gid_t gid; + mode_t mode; +}; + +/* Object methods */ +PyObject *SharedMemory_new(PyTypeObject *, PyObject *, PyObject *); +int SharedMemory_init(SharedMemory *, PyObject *, PyObject *); +void SharedMemory_dealloc(SharedMemory *); +PyObject *SharedMemory_attach(SharedMemory *, PyObject *); +PyObject *SharedMemory_detach(SharedMemory *); +PyObject *SharedMemory_read(SharedMemory *, PyObject *, PyObject *); +PyObject *SharedMemory_write(SharedMemory *, PyObject *, PyObject *); +PyObject *SharedMemory_remove(SharedMemory *); + +/* Object attributes (read-write & read-only) */ + +PyObject *shm_get_uid(SharedMemory *); +int shm_set_uid(SharedMemory *, PyObject *); + +PyObject *shm_get_gid(SharedMemory *); +int shm_set_gid(SharedMemory *, PyObject *); + +PyObject *shm_get_mode(SharedMemory *); +int shm_set_mode(SharedMemory *, PyObject *); + +PyObject *shm_get_key(SharedMemory *); +PyObject *shm_get_size(SharedMemory *); +PyObject *shm_get_address(SharedMemory *); +PyObject *shm_get_attached(SharedMemory *); +PyObject *shm_get_last_attach_time(SharedMemory *); +PyObject *shm_get_last_detach_time(SharedMemory *); +PyObject *shm_get_last_change_time(SharedMemory *); +PyObject *shm_get_creator_pid(SharedMemory *); +PyObject *shm_get_last_pid(SharedMemory *); +PyObject *shm_get_number_attached(SharedMemory *); +PyObject *shm_get_cuid(SharedMemory *); +PyObject *shm_get_cgid(SharedMemory *); + +PyObject *shm_str(SharedMemory *); +PyObject *shm_repr(SharedMemory *); + + +/* Utility functions */ +PyObject *shm_remove(int); + +PyObject *shm_attach(SharedMemory *, int); + diff --git a/mq.c b/mq.c new file mode 100644 index 0000000..b80f790 --- /dev/null +++ b/mq.c @@ -0,0 +1,667 @@ +#include "Python.h" +#include "structmember.h" + +#include "common.h" +#include "mq.h" + + +PyObject * +mq_str(MessageQueue *self) { +#if PY_MAJOR_VERSION > 2 + return PyUnicode_FromFormat("Key=%ld, id=%d", (long)self->key, self->id); +#else + return PyString_FromFormat("Key=%ld, id=%d", (long)self->key, self->id); +#endif +} + + +PyObject * +mq_repr(MessageQueue *self) { +#if PY_MAJOR_VERSION > 2 + return PyUnicode_FromFormat("sysv_ipc.MessageQueue(%ld)", (long)self->key); +#else + return PyString_FromFormat("sysv_ipc.MessageQueue(%ld)", (long)self->key); +#endif +} + + +static PyObject * +get_a_value(int queue_id, enum GET_SET_IDENTIFIERS field) { + struct msqid_ds q_info; + PyObject *py_value = NULL; + + DPRINTF("Calling msgctl(...IPC_STAT...), field = %d\n", field); + if (-1 == msgctl(queue_id, IPC_STAT, &q_info)) { + switch (errno) { + case EIDRM: + case EINVAL: + PyErr_Format(pExistentialException, + "The queue no longer exists"); + break; + + case EACCES: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + + goto error_return; + } + + switch (field) { + case SVIFP_MQ_LAST_SEND_TIME: + py_value = TIME_T_TO_PY(q_info.msg_stime); + break; + + case SVIFP_MQ_LAST_RECEIVE_TIME: + py_value = TIME_T_TO_PY(q_info.msg_rtime); + break; + + case SVIFP_MQ_LAST_CHANGE_TIME: + py_value = TIME_T_TO_PY(q_info.msg_ctime); + break; + + case SVIFP_MQ_CURRENT_MESSAGES: + py_value = MSGQNUM_T_TO_PY(q_info.msg_qnum); + break; + + case SVIFP_MQ_QUEUE_BYTES_MAX: + py_value = MSGLEN_T_TO_PY(q_info.msg_qbytes); + break; + + case SVIFP_MQ_LAST_SEND_PID: + py_value = PID_T_TO_PY(q_info.msg_lspid); + break; + + case SVIFP_MQ_LAST_RECEIVE_PID: + py_value = PID_T_TO_PY(q_info.msg_lrpid); + break; + + case SVIFP_IPC_PERM_UID: + py_value = UID_T_TO_PY(q_info.msg_perm.uid); + break; + + case SVIFP_IPC_PERM_GID: + py_value = GID_T_TO_PY(q_info.msg_perm.gid); + break; + + case SVIFP_IPC_PERM_CUID: + py_value = UID_T_TO_PY(q_info.msg_perm.cuid); + break; + + case SVIFP_IPC_PERM_CGID: + py_value = GID_T_TO_PY(q_info.msg_perm.cgid); + break; + + case SVIFP_IPC_PERM_MODE: + py_value = MODE_T_TO_PY(q_info.msg_perm.mode); + break; + + default: + PyErr_Format(pInternalException, + "Bad field %d passed to get_a_value", field); + goto error_return; + break; + } + + return py_value; + + error_return: + return NULL; +} + + +int +set_a_value(int id, enum GET_SET_IDENTIFIERS field, PyObject *py_value) { + struct msqid_ds mq_info; + +#if PY_MAJOR_VERSION > 2 + if (!PyLong_Check(py_value)) +#else + if (!PyInt_Check(py_value)) +#endif + { + PyErr_Format(PyExc_TypeError, "The attribute must be an integer"); + goto error_return; + } + + /* Here I get the current values associated with the queue. It's + critical to populate sem_info with current values here (rather than + just using the struct filled with whatever garbage it acquired from + being declared on the stack) because the call to msgctl(...IPC_SET...) + below will copy uid, gid and mode to the kernel's data structure. + */ + if (-1 == msgctl(id, IPC_STAT, &mq_info)) { + switch (errno) { + case EACCES: + case EPERM: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + case EINVAL: + PyErr_SetString(pExistentialException, + "The queue no longer exists"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + switch (field) { + case SVIFP_IPC_PERM_UID: +#if PY_MAJOR_VERSION > 2 + mq_info.msg_perm.uid = PyLong_AsLong(py_value); +#else + mq_info.msg_perm.uid = PyInt_AsLong(py_value); +#endif + break; + + case SVIFP_IPC_PERM_GID: +#if PY_MAJOR_VERSION > 2 + mq_info.msg_perm.gid = PyLong_AsLong(py_value); +#else + mq_info.msg_perm.gid = PyInt_AsLong(py_value); +#endif + break; + + case SVIFP_IPC_PERM_MODE: +#if PY_MAJOR_VERSION > 2 + mq_info.msg_perm.mode = PyLong_AsLong(py_value); +#else + mq_info.msg_perm.mode = PyInt_AsLong(py_value); +#endif + break; + + case SVIFP_MQ_QUEUE_BYTES_MAX: + // A msglen_t is unsigned. + // ref: http://www.opengroup.org/onlinepubs/000095399/basedefs/sys/msg.h.html +#if PY_MAJOR_VERSION > 2 + mq_info.msg_qbytes = PyLong_AsUnsignedLongMask(py_value); +#else + mq_info.msg_qbytes = PyInt_AsUnsignedLongMask(py_value); +#endif + break; + + default: + PyErr_Format(pInternalException, + "Bad field %d passed to set_a_value", field); + goto error_return; + break; + } + + if (-1 == msgctl(id, IPC_SET, &mq_info)) { + switch (errno) { + case EACCES: + case EPERM: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + case EINVAL: + PyErr_SetString(pExistentialException, + "The queue no longer exists"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + return 0; + + error_return: + return -1; +} + + +PyObject * +mq_get_key(MessageQueue *self) { + return KEY_T_TO_PY(self->key); +} + +PyObject * +mq_get_last_send_time(MessageQueue *self) { + return get_a_value(self->id, SVIFP_MQ_LAST_SEND_TIME); +} + +PyObject * +mq_get_last_receive_time(MessageQueue *self) { + return get_a_value(self->id, SVIFP_MQ_LAST_RECEIVE_TIME); +} + +PyObject * +mq_get_last_change_time(MessageQueue *self) { + return get_a_value(self->id, SVIFP_MQ_LAST_CHANGE_TIME); +} + +PyObject * +mq_get_last_send_pid(MessageQueue *self) { + return get_a_value(self->id, SVIFP_MQ_LAST_SEND_PID); +} + +PyObject * +mq_get_last_receive_pid(MessageQueue *self) { + return get_a_value(self->id, SVIFP_MQ_LAST_RECEIVE_PID); +} + +PyObject * +mq_get_current_messages(MessageQueue *self) { + return get_a_value(self->id, SVIFP_MQ_CURRENT_MESSAGES); +} + +PyObject * +mq_get_max_size(MessageQueue *self) { + return get_a_value(self->id, SVIFP_MQ_QUEUE_BYTES_MAX); +} + +int +mq_set_max_size(MessageQueue *self, PyObject *py_value) { + return set_a_value(self->id, SVIFP_MQ_QUEUE_BYTES_MAX, py_value); +} + +PyObject * +mq_get_mode(MessageQueue *self) { + return get_a_value(self->id, SVIFP_IPC_PERM_MODE); +} + +int +mq_set_mode(MessageQueue *self, PyObject *py_value) { + return set_a_value(self->id, SVIFP_IPC_PERM_MODE, py_value); +} + +PyObject * +mq_get_uid(MessageQueue *self) { + return get_a_value(self->id, SVIFP_IPC_PERM_UID); +} + +int +mq_set_uid(MessageQueue *self, PyObject *py_value) { + return set_a_value(self->id, SVIFP_IPC_PERM_UID, py_value); +} + +PyObject * +mq_get_gid(MessageQueue *self) { + return get_a_value(self->id, SVIFP_IPC_PERM_GID); +} + +int +mq_set_gid(MessageQueue *self, PyObject *py_value) { + return set_a_value(self->id, SVIFP_IPC_PERM_GID, py_value); +} + +PyObject * +mq_get_c_uid(MessageQueue *self) { + return get_a_value(self->id, SVIFP_IPC_PERM_CUID); +} + +PyObject * +mq_get_c_gid(MessageQueue *self) { + return get_a_value(self->id, SVIFP_IPC_PERM_CGID); +} + + +PyObject * +mq_remove(int queue_id) { + struct msqid_ds mq_info; + + DPRINTF("calling msgctl(...IPC_RMID...) on id %d\n", queue_id); + if (-1 == msgctl(queue_id, IPC_RMID, &mq_info)) { + switch (errno) { + case EIDRM: + case EINVAL: + PyErr_Format(pExistentialException, + "The queue no longer exists"); + break; + + case EPERM: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + + +void +MessageQueue_dealloc(MessageQueue *self) { + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject * +MessageQueue_new(PyTypeObject *type, PyObject *args, PyObject *keywords) { + MessageQueue *self; + + self = (MessageQueue *)type->tp_alloc(type, 0); + + return (PyObject *)self; +} + + +int +MessageQueue_init(MessageQueue *self, PyObject *args, PyObject *keywords) { + int flags = 0; + int mode = 0600; + NoneableKey key; + unsigned long max_message_size = QUEUE_MESSAGE_SIZE_MAX_DEFAULT; + char *keyword_list[ ] = {"key", "flags", "mode", "max_message_size", NULL}; + + //MessageQueue(key, [flags = 0, [mode = 0600, [max_message_size = QUEUE_MESSAGE_SIZE_MAX_DEFAULT]]) + + if (!PyArg_ParseTupleAndKeywords(args, keywords, "O&|iik", keyword_list, + convert_key_param, &key, &flags, + &mode, &max_message_size)) + goto error_return; + + if (max_message_size > QUEUE_MESSAGE_SIZE_MAX) { + PyErr_Format(PyExc_ValueError, "The message length must be <= %lu\n", + (unsigned long)QUEUE_MESSAGE_SIZE_MAX); + goto error_return; + } + + if ( !(flags & IPC_CREAT) && (flags & IPC_EXCL) ) { + PyErr_SetString(PyExc_ValueError, + "IPC_EXCL must be combined with IPC_CREAT"); + goto error_return; + } + + if (key.is_none && ((flags & IPC_EXCL) != IPC_EXCL)) { + PyErr_SetString(PyExc_ValueError, + "Key can only be None if IPC_EXCL is set"); + goto error_return; + } + + self->max_message_size = max_message_size; + + // I mask the caller's flags against the two IPC_* flags to ensure that + // nothing funky sneaks into the flags. + flags &= (IPC_CREAT | IPC_EXCL); + + mode &= 0777; + + if (key.is_none) { + // (key == None) ==> generate a key for the caller + do { + errno = 0; + self->key = get_random_key(); + + DPRINTF("Calling msgget, key=%ld, flags=0x%x\n", + (long)self->key, flags); + self->id = msgget(self->key, mode | flags); + } while ( (-1 == self->id) && (EEXIST == errno) ); + } + else { + // (key != None) ==> use key supplied by the caller + self->key = key.value; + + DPRINTF("Calling msgget, key=%ld, flags=0x%x\n", (long)self->key, flags); + self->id = msgget(self->key, mode | flags); + } + + DPRINTF("id == %d\n", self->id); + + if (self->id == -1) { + switch (errno) { + case EACCES: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + case EEXIST: + PyErr_SetString(pExistentialException, + "A queue with the specified key already exists"); + break; + + case ENOENT: + PyErr_SetString(pExistentialException, + "No queue exists with the specified key"); + break; + + case ENOMEM: + PyErr_SetString(PyExc_MemoryError, "Not enough memory"); + break; + + case ENOSPC: + PyErr_SetString(PyExc_OSError, + "The system limit for message queues has been reached"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + goto error_return; + } + + return 0; + + error_return: + return -1; +} + + +PyObject * +MessageQueue_send(MessageQueue *self, PyObject *args, PyObject *keywords) { + /* In Python >= 2.5, the Python argument specifier 's#' expects a + py_ssize_t for its second parameter. A ulong is long enough to hold + a py_ssize_t. + It might be too big, though, on platforms where a long is larger than + py_ssize_t. Therefore I *must* initialize it to 0 so that whatever + Python doesn't write to is zeroed out. + */ +#if PY_MAJOR_VERSION > 2 + static char args_format[] = "s*|Oi"; + Py_buffer user_msg; +#else + static char args_format[] = "s#|Oi"; + typedef struct { + char *buf; + long len; + } MyBuffer; + MyBuffer user_msg; + user_msg.len = 0; +#endif + PyObject *py_block = NULL; + int flags = 0; + int type = 1; + int rc; + struct queue_message *p_msg = NULL; + char *keyword_list[ ] = {"message", "block", "type", NULL}; + + // send(message, [block = True, [type = 1]]) + if (!PyArg_ParseTupleAndKeywords(args, keywords, args_format, keyword_list, +#if PY_MAJOR_VERSION > 2 + &user_msg, +#else + &(user_msg.buf), &(user_msg.len), +#endif + &py_block, &type)) + goto error_return; + + if (type <= 0) { + PyErr_SetString(PyExc_ValueError, "The type must be > 0"); + goto error_return; + } + + if (user_msg.len > self->max_message_size) { + PyErr_Format(PyExc_ValueError, + "The message length exceeds queue's max_message_size (%lu)", + self->max_message_size); + goto error_return; + } + // default behavior (when py_block == NULL) is to block/wait. + if (py_block && PyObject_Not(py_block)) + flags |= IPC_NOWAIT; + + p_msg = (struct queue_message *)malloc(offsetof(struct queue_message, message) + user_msg.len); + + DPRINTF("p_msg is %p\n", p_msg); + + if (!p_msg) { + PyErr_SetString(PyExc_MemoryError, "Out of memory"); + goto error_return; + } + + memcpy(p_msg->message, user_msg.buf, user_msg.len); + p_msg->type = type; + + Py_BEGIN_ALLOW_THREADS + DPRINTF("Calling msgsnd(), id=%ld, p_msg=%p, length=%lu, flags=0x%x\n", + (long)self->id, p_msg, user_msg.len, flags); + rc = msgsnd(self->id, p_msg, (size_t)user_msg.len, flags); + Py_END_ALLOW_THREADS + + if (-1 == rc) { + switch (errno) { + case EACCES: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + case EAGAIN: + PyErr_SetString(pBusyException, + "The queue is full, or a system-wide limit on the number of queue messages has been reached"); + break; + + case EIDRM: + PyErr_SetString(pExistentialException, + "The queue no longer exists"); + break; + + case EINTR: + PyErr_SetString(pBaseException, "Signaled while waiting"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + + goto error_return; + } + + +#if PY_MAJOR_VERSION > 2 + PyBuffer_Release(&user_msg); +#endif + + free(p_msg); + + Py_RETURN_NONE; + + error_return: +#if PY_MAJOR_VERSION > 2 + PyBuffer_Release(&user_msg); +#endif + free(p_msg); + return NULL; +} + + +PyObject * +MessageQueue_receive(MessageQueue *self, PyObject *args, PyObject *keywords) { + PyObject *py_block = NULL; + PyObject *py_return_tuple = NULL; + int flags = 0; + int type = 0; + ssize_t rc; + struct queue_message *p_msg = NULL; + char *keyword_list[ ] = {"block", "type", NULL}; + + // receive([block = True, [type = 0]]) + if (!PyArg_ParseTupleAndKeywords(args, keywords, "|Oi", keyword_list, + &py_block, &type)) + goto error_return; + + // default behavior (when py_block == NULL) is to block/wait. + if (py_block && PyObject_Not(py_block)) + flags |= IPC_NOWAIT; + + p_msg = (struct queue_message *)malloc(sizeof(struct queue_message) + self->max_message_size); + + DPRINTF("p_msg is %p, size = %lu\n", + p_msg, sizeof(struct queue_message) + self->max_message_size); + + if (!p_msg) { + PyErr_SetString(PyExc_MemoryError, "Out of memory"); + goto error_return; + } + + p_msg->type = type; + + Py_BEGIN_ALLOW_THREADS; + rc = msgrcv(self->id, p_msg, (size_t)self->max_message_size, + type, flags); + Py_END_ALLOW_THREADS; + + DPRINTF("after msgrcv, p_msg->type=%ld, rc (size)=%ld\n", + p_msg->type, (long)rc); + + if ((ssize_t)-1 == rc) { + switch (errno) { + case EACCES: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + case EIDRM: + case EINVAL: + PyErr_SetString(pExistentialException, + "The queue no longer exists"); + break; + + case EINTR: + PyErr_SetString(pBaseException, "Signaled while waiting"); + break; + + case ENOMSG: + PyErr_SetString(pBusyException, + "No available messages of the specified type"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + + goto error_return; + } + + py_return_tuple = Py_BuildValue("NN", +#if PY_MAJOR_VERSION > 2 + PyBytes_FromStringAndSize(p_msg->message, rc), + PyLong_FromLong(p_msg->type) +#else + PyString_FromStringAndSize(p_msg->message, rc), + PyInt_FromLong(p_msg->type) +#endif + ); + + free(p_msg); + + return py_return_tuple; + + error_return: + free(p_msg); + return NULL; +} + + +PyObject * +MessageQueue_remove(MessageQueue *self) { + return mq_remove(self->id); +} + diff --git a/mq.h b/mq.h new file mode 100644 index 0000000..b5af78f --- /dev/null +++ b/mq.h @@ -0,0 +1,79 @@ +#include // for definition of SSIZE_MAX + +typedef struct { + PyObject_HEAD + key_t key; + int id; + unsigned long max_message_size; +} MessageQueue; + +/* Message queue message struct for send() & receive() +On many systems this is defined in sys/msg.h already, but it's better +for me to define it here. Name it something other than msgbuf to avoid +conflict with the struct that the OS header file might define. +*/ +struct queue_message { + long type; + char message[]; +}; + +/* Maximum message size is limited by (a) the largest Python string I can +create and (b) SSIZE_T_MAX. The latter restriction comes from the spec which +says, "If the value of msgsz is greater than {SSIZE_MAX}, the result is +implementation-defined." +ref: http://www.opengroup.org/onlinepubs/000095399/functions/msgrcv.html +*/ +#if SSIZE_MAX > PY_STRING_LENGTH_MAX +#define QUEUE_MESSAGE_SIZE_MAX PY_STRING_LENGTH_MAX +#else +#define QUEUE_MESSAGE_SIZE_MAX SSIZE_MAX +#endif + +/* The max message size is probably a very big number, and since a +max-sized buffer is allocated every time receive() is called, it would be +ugly if the default message size for new queues was the same as the max. +In addition, many operating systems limit the entire queue to 2048 bytes, +so defaulting the max message to something larger seems a bit stupid. + +This value is also present in numeric form in ReadMe.html, so if you +change it here, change it there too. +*/ +#define QUEUE_MESSAGE_SIZE_MAX_DEFAULT 2048 + +/* Object methods */ +PyObject *MessageQueue_new(PyTypeObject *, PyObject *, PyObject *); +int MessageQueue_init(MessageQueue *, PyObject *, PyObject *); +void MessageQueue_dealloc(MessageQueue *); +PyObject *MessageQueue_send(MessageQueue *, PyObject *, PyObject *); +PyObject *MessageQueue_receive(MessageQueue *, PyObject *, PyObject *); +PyObject *MessageQueue_remove(MessageQueue *); + +/* Object attributes (read-write & read-only) */ +PyObject *mq_get_mode(MessageQueue *); +int mq_set_mode(MessageQueue *, PyObject *); + +PyObject *mq_get_uid(MessageQueue *); +int mq_set_uid(MessageQueue *, PyObject *); + +PyObject *mq_get_gid(MessageQueue *); +int mq_set_gid(MessageQueue *, PyObject *); + +PyObject *mq_get_max_size(MessageQueue *); +int mq_set_max_size(MessageQueue *, PyObject *); + +PyObject *mq_get_key(MessageQueue *); +PyObject *mq_get_last_send_time(MessageQueue *); +PyObject *mq_get_last_receive_time(MessageQueue *); +PyObject *mq_get_last_change_time(MessageQueue *); +PyObject *mq_get_last_send_pid(MessageQueue *); +PyObject *mq_get_last_receive_pid(MessageQueue *); +PyObject *mq_get_current_messages(MessageQueue *); +PyObject *mq_get_c_uid(MessageQueue *); +PyObject *mq_get_c_gid(MessageQueue *); + +PyObject *mq_str(MessageQueue *); +PyObject *mq_repr(MessageQueue *); + +/* Misc. */ +PyObject *mq_remove(int); + diff --git a/prober.py b/prober.py new file mode 100644 index 0000000..08f5fe0 --- /dev/null +++ b/prober.py @@ -0,0 +1,186 @@ +import os.path +import os +import subprocess +import sys +import distutils.sysconfig + +# Set these to None for debugging or subprocess.PIPE to silence compiler +# warnings and errors. +STDOUT = subprocess.PIPE +STDERR = subprocess.PIPE +# STDOUT = None +# STDERR = None + +# This is the max length that I want a printed line to be. +MAX_LINE_LENGTH = 78 + +PYTHON_INCLUDE_DIR = os.path.dirname(distutils.sysconfig.get_config_h_filename()) +#print (PYTHON_INCLUDE_DIR) + +def line_wrap_paragraph(s): + # Format s with terminal-friendly line wraps. + done = False + beginning = 0 + end = MAX_LINE_LENGTH - 1 + lines = [ ] + while not done: + if end >= len(s): + done = True + lines.append(s[beginning:]) + else: + last_space = s[beginning:end].rfind(' ') + + lines.append(s[beginning:beginning + last_space]) + beginning += (last_space + 1) + end = beginning + MAX_LINE_LENGTH - 1 + + return lines + + +def print_bad_news(value_name, default): + s = "Setup can't determine %s on your system, so it will default to %s which may not be correct." \ + % (value_name, default) + plea = "Please report this message and your operating system info to the package maintainer listed in the README file." + + lines = line_wrap_paragraph(s) + [''] + line_wrap_paragraph(plea) + + border = '*' * MAX_LINE_LENGTH + + s = border + "\n* " + ('\n* '.join(lines)) + '\n' + border + + print (s) + + +def does_build_succeed(filename): + # Utility function that returns True if the file compiles and links + # successfully, False otherwise. + cmd = "cc -Wall -I%s -o ./prober/foo ./prober/%s" % \ + (PYTHON_INCLUDE_DIR, filename) + + p = subprocess.Popen(cmd, shell=True, stdout=STDOUT, stderr=STDERR) + + # p.wait() returns the process' return code, so 0 implies that + # the compile & link succeeded. + return not bool(p.wait()) + +def compile_and_run(filename, linker_options = ""): + # Utility function that returns the stdout output from running the + # compiled source file; None if the compile fails. + cmd = "cc -Wall -I%s -o ./prober/foo %s ./prober/%s" % \ + (PYTHON_INCLUDE_DIR, linker_options, filename) + + p = subprocess.Popen(cmd, shell=True, stdout=STDOUT, stderr=STDERR) + + if p.wait(): + # uh-oh, compile failed + return None + else: + s = subprocess.Popen(["./prober/foo"], + stdout=subprocess.PIPE).communicate()[0] + return s.strip().decode() + + +def sniff_semtimedop(): + return does_build_succeed("semtimedop_test.c") + + +def sniff_union_semun_defined(): + # AFAICT the semun union is supposed to be declared in one's code. + # However, a lot of legacy code gets this wrong and some header files + # define it, e.g.sys/sem.h on OS X where it's #ifdef-ed so that legacy + # code won't break. On some systems, it appears and disappears based + # on the #define value of _XOPEN_SOURCE. + return does_build_succeed("sniff_union_semun_defined.c") + + +def probe_semvmx(): + # This is the hardcoded default that I chose for two reasons. First, + # it's the default on my Mac so I know at least one system needs it + # this low. Second, it fits neatly into a 16-bit signed int which + # makes me hope that it's low enough to be safe on all systems. + semvmx = 32767 + + # FIXME -- Ways to get SEMVMX -- + # 1) Try to compile .c code assuming SEMVMX is #defined + # 2) Parse the output from `sysctl kern.sysv.semvmx` (doesn't work + # on OS X). + # 3) Parse the output from `ipcs -S` + # 4) Run .c code that loops, releasing a semaphore until semop() + # returns ERANGE or the value goes wonky. OS X lets the latter + # happen, which AFAICT is a violation of the spec. + + return semvmx + + + +def probe_page_size(): + DEFAULT_PAGE_SIZE = 4096 + + page_size = compile_and_run("probe_page_size.c") + + if page_size is None: + page_size = DEFAULT_PAGE_SIZE + print_bad_news("the value of PAGE_SIZE", page_size) + + return page_size + + +def probe(): + d = { "KEY_MAX" : "LONG_MAX", + "KEY_MIN" : "LONG_MIN" + } + + conditionals = [ "_SEM_SEMUN_UNDEFINED" ] + + f = open("VERSION") + version = f.read().strip() + f.close() + + d["SYSV_IPC_VERSION"] = '"%s"' % version + d["PAGE_SIZE"] = probe_page_size() + if sniff_semtimedop(): + d["SEMTIMEDOP_EXISTS"] = "" + d["SEMAPHORE_VALUE_MAX"] = probe_semvmx() + # Some (all?) Linux platforms #define _SEM_SEMUN_UNDEFINED if it's up + # to my code to declare this union, so I use that flag as my standard. + if not sniff_union_semun_defined(): + d["_SEM_SEMUN_UNDEFINED"] = "" + + + + msg = """/* +This header file was generated when you ran setup. Once created, the setup +process won't overwrite it, so you can adjust the values by hand and +recompile if you need to. + +To recreate this file, just delete it and re-run setup.py. + +KEY_MIN, KEY_MAX and SEMAPHORE_VALUE_MAX are stored internally in longs, so +you should never #define them to anything larger than LONG_MAX. +*/ + +""" + + filename = "probe_results.h" + if not os.path.exists(filename): + lines = [ ] + + for key in d: + if key in conditionals: + lines.append("#ifndef %s" % key) + + lines.append("#define %s\t\t%s" % (key, d[key])) + + if key in conditionals: + lines.append("#endif") + + # A trailing '\n' keeps compilers happy... + f = open(filename, "w") + f.write(msg + '\n'.join(lines) + '\n') + f.close() + + return d + +if __name__ == "__main__": + s = probe() + print (s) diff --git a/prober/probe_page_size.c b/prober/probe_page_size.c new file mode 100644 index 0000000..bc72bc1 --- /dev/null +++ b/prober/probe_page_size.c @@ -0,0 +1,22 @@ +//#define _XOPEN_SOURCE 500 +#include "Python.h" + +#include + +// Code for determining page size swiped from Python's mmapmodule.c +#if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE) +static int +my_getpagesize(void) +{ + return sysconf(_SC_PAGESIZE); +} +#else +#include +#define my_getpagesize getpagesize +#endif + +int main(void) { + printf("%d\n", my_getpagesize()); + + return 0; +} diff --git a/prober/semtimedop_test.c b/prober/semtimedop_test.c new file mode 100644 index 0000000..e165629 --- /dev/null +++ b/prober/semtimedop_test.c @@ -0,0 +1,11 @@ +//#define _XOPEN_SOURCE 500 +#include "Python.h" + +#include +#include + +int main(void) { + semtimedop(0, NULL, 0, NULL); + + return 0; +} diff --git a/prober/sniff_union_semun_defined.c b/prober/sniff_union_semun_defined.c new file mode 100644 index 0000000..8ec64eb --- /dev/null +++ b/prober/sniff_union_semun_defined.c @@ -0,0 +1,12 @@ +//#define _XOPEN_SOURCE 500 +#include "Python.h" + +#include + +int main(void) { + union semun foo; + + foo.val = 42; + + return 0; +} diff --git a/semaphore.c b/semaphore.c new file mode 100644 index 0000000..57cf2aa --- /dev/null +++ b/semaphore.c @@ -0,0 +1,776 @@ +#include "Python.h" +#include "structmember.h" + +#include "common.h" +#include "semaphore.h" + +#define ONE_BILLION 1000000000 + +// This enum has to start at zero because its values are used as an +// arry index in sem_perform_semop(). +enum SEMOP_TYPE { + SEMOP_P = 0, + SEMOP_V, + SEMOP_Z +}; + +/* Struct to contain a timeout which can be None */ +typedef struct { + int is_none; + int is_zero; + struct timespec timestamp; +} NoneableTimeout; + + +// It is recommended practice to define this union here in the .c module, but +// it's been common practice for platforms to define it themselves in header +// files. For instance, BSD and OS X do so (provisionally) in sem.h. As a +// result, I need to surround this with an #ifdef. +#ifdef _SEM_SEMUN_UNDEFINED +union semun { + int val; /* used for SETVAL only */ + struct semid_ds *buf; /* for IPC_STAT and IPC_SET */ + unsigned short *array; /* used for GETALL and SETALL */ +}; +#endif + + +static int +convert_timeout(PyObject *py_timeout, void *converted_timeout) { + // Converts a PyObject into a timeout if possible. The PyObject should + // be None or some sort of numeric value (e.g. int, float, etc.) + // converted_timeout should point to a NoneableTimeout. When this function + // returns, if the NoneableTimeout's is_none is true, then the rest of the + // struct is undefined. Otherwise, the rest of the struct is populated. + int rc = 0; + double simple_timeout = 0; + NoneableTimeout *p_timeout = (NoneableTimeout *)converted_timeout; + + // The timeout can be None or any Python numeric type (float, + // int, long). + if (py_timeout == Py_None) + rc = 1; + else if (PyFloat_Check(py_timeout)) { + rc = 1; + simple_timeout = PyFloat_AsDouble(py_timeout); + } +#if PY_MAJOR_VERSION < 3 + else if (PyInt_Check(py_timeout)) { + rc = 1; + simple_timeout = (double)PyInt_AsLong(py_timeout); + } +#endif + else if (PyLong_Check(py_timeout)) { + rc = 1; + simple_timeout = (double)PyLong_AsLong(py_timeout); + } + + // The timeout may not be negative. + if ((rc) && (simple_timeout < 0)) + rc = 0; + + if (!rc) + PyErr_SetString(PyExc_TypeError, + "The timeout must be None or a non-negative number"); + else { + if (py_timeout == Py_None) + p_timeout->is_none = 1; + else { + p_timeout->is_none = 0; + + p_timeout->is_zero = (!simple_timeout); + + // Note the difference between this and POSIX timeouts. System V + // timeouts expect tv_sec to represent a delta from the current + // time whereas POSIX semaphores expect an absolute value. + p_timeout->timestamp.tv_sec = (time_t)floor(simple_timeout); + p_timeout->timestamp.tv_nsec = (long)((simple_timeout - floor(simple_timeout)) * ONE_BILLION); + } + } + + return rc; +} + + +PyObject * +sem_str(Semaphore *self) { +#if PY_MAJOR_VERSION > 2 + return PyUnicode_FromFormat("Key=%ld, id=%d", (long)self->key, self->id); +#else + return PyString_FromFormat("Key=%ld, id=%d", (long)self->key, self->id); +#endif +} + + +PyObject * +sem_repr(Semaphore *self) { +#if PY_MAJOR_VERSION > 2 + return PyUnicode_FromFormat("sysv_ipc.Semaphore(%ld)", (long)self->key); +#else + return PyString_FromFormat("sysv_ipc.Semaphore(%ld)", (long)self->key); +#endif +} + + +static void +sem_set_error(void) { + switch (errno) { + case ENOENT: + case EINVAL: + PyErr_SetString(pExistentialException, + "No semaphore exists with the specified key"); + break; + + case EEXIST: + PyErr_SetString(pExistentialException, + "A semaphore with the specified key already exists"); + break; + + case EACCES: + PyErr_SetString(pPermissionsException, "Permission denied"); + break; + + case ERANGE: + PyErr_Format(PyExc_ValueError, + "The semaphore's value must remain between 0 and %ld (SEMAPHORE_VALUE_MAX)", + (long)SEMAPHORE_VALUE_MAX); + break; + + case EAGAIN: + PyErr_SetString(pBusyException, "The semaphore is busy"); + break; + + case EIDRM: + PyErr_SetString(pExistentialException, "The semaphore was removed"); + break; + + case EINTR: + PyErr_SetString(pBaseException, "Signaled while waiting"); + break; + + case ENOMEM: + PyErr_SetString(PyExc_MemoryError, "Not enough memory"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } +} + + +static PyObject * +sem_perform_semop(enum SEMOP_TYPE op_type, Semaphore *self, PyObject *args, PyObject *keywords) { + int rc = 0; + NoneableTimeout timeout; + struct sembuf op[1]; + /* delta (a.k.a. struct sembuf.sem_op) is a short + ref: http://www.opengroup.org/onlinepubs/000095399/functions/semop.html + */ + short int delta; + char *keyword_list[3][3] = { + {"timeout", "delta", NULL}, // P == acquire + {"delta", NULL}, // V == release + {"timeout", NULL} // Z == zero test + }; + + + /* Initialize this to the default value. If the user doesn't pass a + timeout, Python won't call convert_timeout() and so the timeout + will be otherwise uninitialized. + */ + timeout.is_none = 1; + + /* op_type is P, V or Z corresponding to the 3 Semaphore methods + that call that call semop(). */ + switch (op_type) { + case SEMOP_P: + // P == acquire + delta = -1; + rc = PyArg_ParseTupleAndKeywords(args, keywords, "|O&h", + keyword_list[SEMOP_P], + convert_timeout, &timeout, + &delta); + + if (rc && !delta) { + rc = 0; + PyErr_SetString(PyExc_ValueError, "The delta must be non-zero"); + } + else + delta = -abs(delta); + break; + + case SEMOP_V: + // V == release + delta = 1; + rc = PyArg_ParseTupleAndKeywords(args, keywords, "|h", + keyword_list[SEMOP_V], + &delta); + + if (rc && !delta) { + rc = 0; + PyErr_SetString(PyExc_ValueError, "The delta must be non-zero"); + } + else + delta = abs(delta); + break; + + case SEMOP_Z: + // Z = Zero test + delta = 0; + rc = PyArg_ParseTupleAndKeywords(args, keywords, "|O&", + keyword_list[SEMOP_Z], + convert_timeout, &timeout); + break; + + default: + PyErr_Format(pInternalException, "Bad op_type (%d)", op_type); + rc = 0; + break; + } + + if (!rc) + goto error_return; + + // Now that the caller's params have been vetted, I set up the op struct + // that I'm going to pass to semop(). + op[0].sem_num = 0; + op[0].sem_op = delta; + op[0].sem_flg = self->op_flags; + + Py_BEGIN_ALLOW_THREADS; +#ifdef SEMTIMEDOP_EXISTS + // Call semtimedop() if appropriate, otherwise call semop() + if (!timeout.is_none) { + DPRINTF("calling semtimedop on id %d, op.sem_op=%d, op.flags=0x%x\n", + self->id, op[0].sem_op, op[0].sem_flg); + DPRINTF("timeout tv_sec = %ld; timeout tv_nsec = %ld\n", + timeout.timestamp.tv_sec, timeout.timestamp.tv_nsec); + rc = semtimedop(self->id, op, 1, &timeout.timestamp); + } + else { + DPRINTF("calling semop on id %d, op.sem_op = %d, op.flags=%x\n", + self->id, op[0].sem_op, op[0].sem_flg); + rc = semop(self->id, op, 1); + } +#else + // no support for semtimedop(), always call semop() instead. + DPRINTF("calling semop on id %d, op.sem_op = %d, op.flags=%x\n", + self->id, op[0].sem_op, op[0].sem_flg); + rc = semop(self->id, op, 1); +#endif + Py_END_ALLOW_THREADS; + + if (rc == -1) { + sem_set_error(); + goto error_return; + } + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + +// cmd can be any of the values defined in the documentation for semctl(). +static PyObject * +sem_get_semctl_value(int semaphore_id, int cmd) { + int rc; + + // semctl() returns an int + // ref: http://www.opengroup.org/onlinepubs/000095399/functions/semctl.html + rc = semctl(semaphore_id, 0, cmd); + + if (-1 == rc) { + sem_set_error(); + goto error_return; + } + +#if PY_MAJOR_VERSION > 2 + return PyLong_FromLong(rc); +#else + return PyInt_FromLong(rc); +#endif + + error_return: + return NULL; +} + + +static PyObject * +sem_get_ipc_perm_value(int id, enum GET_SET_IDENTIFIERS field) { + struct semid_ds sem_info; + union semun arg; + PyObject *py_value = NULL; + + arg.buf = &sem_info; + + // Here I get the values currently associated with the semaphore. + if (-1 == semctl(id, 0, IPC_STAT, arg)) { + sem_set_error(); + goto error_return; + } + + switch (field) { + case SVIFP_IPC_PERM_UID: + py_value = UID_T_TO_PY(sem_info.sem_perm.uid); + break; + + case SVIFP_IPC_PERM_GID: + py_value = GID_T_TO_PY(sem_info.sem_perm.gid); + break; + + case SVIFP_IPC_PERM_CUID: + py_value = UID_T_TO_PY(sem_info.sem_perm.cuid); + break; + + case SVIFP_IPC_PERM_CGID: + py_value = GID_T_TO_PY(sem_info.sem_perm.cgid); + break; + + case SVIFP_IPC_PERM_MODE: + py_value = MODE_T_TO_PY(sem_info.sem_perm.mode); + break; + + // This isn't an ipc_perm value but it fits here anyway. + case SVIFP_SEM_OTIME: + py_value = TIME_T_TO_PY(sem_info.sem_otime); + break; + + default: + PyErr_Format(pInternalException, + "Bad field %d passed to sem_get_ipc_perm_value", field); + goto error_return; + break; + } + + return py_value; + + error_return: + return NULL; +} + + +static int +sem_set_ipc_perm_value(int id, enum GET_SET_IDENTIFIERS field, PyObject *py_value) { + struct semid_ds sem_info; + union semun arg; + + arg.buf = &sem_info; + +#if PY_MAJOR_VERSION > 2 + if (!PyLong_Check(py_value)) +#else + if (!PyInt_Check(py_value)) +#endif + { + PyErr_Format(PyExc_TypeError, "The attribute must be an integer"); + goto error_return; + } + + arg.buf = &sem_info; + + /* Here I get the current values associated with the semaphore. It's + critical to populate sem_info with current values here (rather than + just using the struct filled with whatever garbage it acquired from + being declared on the stack) because the call to semctl(...IPC_SET...) + below will copy uid, gid and mode to the kernel's data structure. + */ + if (-1 == semctl(id, 0, IPC_STAT, arg)) { + sem_set_error(); + goto error_return; + } + + // Below I'm stuffing a Python int converted to a C long into a + // uid_t, gid_t or mode_t. A long might not fit, hence the explicit + // cast. If the user passes a value that's too big, tough cookies. + switch (field) { + case SVIFP_IPC_PERM_UID: +#if PY_MAJOR_VERSION > 2 + sem_info.sem_perm.uid = (uid_t)PyLong_AsLong(py_value); +#else + sem_info.sem_perm.uid = (uid_t)PyInt_AsLong(py_value); +#endif + break; + + case SVIFP_IPC_PERM_GID: +#if PY_MAJOR_VERSION > 2 + sem_info.sem_perm.gid = (gid_t)PyLong_AsLong(py_value); +#else + sem_info.sem_perm.gid = (gid_t)PyInt_AsLong(py_value); +#endif + break; + + case SVIFP_IPC_PERM_MODE: +#if PY_MAJOR_VERSION > 2 + sem_info.sem_perm.mode = (mode_t)PyLong_AsLong(py_value); +#else + sem_info.sem_perm.mode = (mode_t)PyInt_AsLong(py_value); +#endif + break; + + default: + PyErr_Format(pInternalException, + "Bad field %d passed to sem_set_ipc_perm_value", field); + goto error_return; + break; + } + + if (-1 == semctl(id, 0, IPC_SET, arg)) { + sem_set_error(); + goto error_return; + } + + return 0; + + error_return: + return -1; +} + + +PyObject * +sem_remove(int id) { + if (NULL == sem_get_semctl_value(id, IPC_RMID)) + return NULL; + else + Py_RETURN_NONE; +} + + +void +Semaphore_dealloc(Semaphore *self) { + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject * +Semaphore_new(PyTypeObject *type, PyObject *args, PyObject *keywords) { + Semaphore *self; + + self = (Semaphore *)type->tp_alloc(type, 0); + + return (PyObject *)self; +} + + +int +Semaphore_init(Semaphore *self, PyObject *args, PyObject *keywords) { + int mode = 0600; + int initial_value = 0; + int flags = 0; + union semun arg; + char *keyword_list[ ] = {"key", "flags", "mode", "initial_value", NULL}; + NoneableKey key; + + //Semaphore(key, [flags = 0, [mode = 0600, [initial_value = 0]]]) + + if (!PyArg_ParseTupleAndKeywords(args, keywords, "O&|iii", keyword_list, + &convert_key_param, &key, &flags, + &mode, &initial_value)) + goto error_return; + + DPRINTF("key is none = %d, key value = %ld\n", key.is_none, (long)key.value); + + if ( !(flags & IPC_CREAT) && (flags & IPC_EXCL) ) { + PyErr_SetString(PyExc_ValueError, + "IPC_EXCL must be combined with IPC_CREAT"); + goto error_return; + } + + if (key.is_none && ((flags & IPC_EXCL) != IPC_EXCL)) { + PyErr_SetString(PyExc_ValueError, + "Key can only be None if IPC_EXCL is set"); + goto error_return; + } + + self->op_flags = 0; + + // I mask the caller's flags against the two IPC_* flags to ensure that + // nothing funky sneaks into the flags. + flags &= (IPC_CREAT | IPC_EXCL); + + // Note that Sys V sems can be in "sets" (arrays) but I hardcode this + // to always be a set with just one member. + // Permissions and flags (i.e. IPC_CREAT | IPC_EXCL) are both crammed + // into the 3rd param. + if (key.is_none) { + // (key == None) ==> generate a key for the caller + do { + errno = 0; + self->key = get_random_key(); + + DPRINTF("Calling semget, key=%ld, mode=%o, flags=%x\n", + (long)self->key, mode, flags); + self->id = semget(self->key, 1, mode | flags); + } while ( (-1 == self->id) && (EEXIST == errno) ); + } + else { + // (key != None) ==> use key supplied by the caller + self->key = key.value; + + DPRINTF("Calling semget, key=%ld, mode=%o, flags=%x\n", + (long)self->key, mode, flags); + self->id = semget(self->key, 1, mode | flags); + } + + DPRINTF("id == %d\n", self->id); + + if (self->id == -1) { + sem_set_error(); + goto error_return; + } + + // Before attempting to set the initial value, I have to be sure that + // I created this semaphore and that I have write access to it. + if ((flags & IPC_CREX) && (mode & 0200)) { + DPRINTF("setting initial value to %d\n", initial_value); + arg.val = initial_value; + + if (-1 == semctl(self->id, 0, SETVAL, arg)) { + sem_set_error(); + goto error_return; + } + } + + return 0; + + error_return: + return -1; +} + + +PyObject * +Semaphore_P(Semaphore *self, PyObject *args, PyObject *keywords) { + return sem_perform_semop(SEMOP_P, self, args, keywords); +} + + +PyObject * +Semaphore_acquire(Semaphore *self, PyObject *args, PyObject *keywords) { + return Semaphore_P(self, args, keywords); +} + + +PyObject * +Semaphore_V(Semaphore *self, PyObject *args, PyObject *keywords) { + return sem_perform_semop(SEMOP_V, self, args, keywords); +} + + +PyObject * +Semaphore_release(Semaphore *self, PyObject *args, PyObject *keywords) { + return Semaphore_V(self, args, keywords); +} + + +PyObject * +Semaphore_Z(Semaphore *self, PyObject *args, PyObject *keywords) { + return sem_perform_semop(SEMOP_Z, self, args, keywords); +} + + +PyObject * +Semaphore_remove(Semaphore *self) { + return sem_remove(self->id); +} + +PyObject * +Semaphore_enter(Semaphore *self) { + PyObject *args = PyTuple_New(0); + PyObject *retval = NULL; + + if (Semaphore_acquire(self, args, NULL)) { + retval = (PyObject *)self; + Py_INCREF(self); + } + + Py_DECREF(args); + + return retval; +} + +PyObject * +Semaphore_exit(Semaphore *self, PyObject *args) { + PyObject *release_args = PyTuple_New(0); + PyObject *retval = NULL; + + DPRINTF("exiting context and releasing semaphore %s\n", self->key); + + retval = Semaphore_release(self, release_args, NULL); + + Py_DECREF(release_args); + + return retval; +} + +PyObject * +sem_get_key(Semaphore *self) { + return KEY_T_TO_PY(self->key); +} + +PyObject * +sem_get_value(Semaphore *self) { + return sem_get_semctl_value(self->id, GETVAL); +} + + +int +sem_set_value(Semaphore *self, PyObject *py_value) +{ + union semun arg; + long value; + +#if PY_MAJOR_VERSION > 2 + if (!PyLong_Check(py_value)) +#else + if (!PyInt_Check(py_value)) +#endif + { + PyErr_Format(PyExc_TypeError, "Attribute 'value' must be an integer"); + goto error_return; + } + +#if PY_MAJOR_VERSION > 2 + value = PyLong_AsLong(py_value); +#else + value = PyInt_AsLong(py_value); +#endif + + DPRINTF("C value is %ld\n", value); + + if ((-1 == value) && PyErr_Occurred()) { + // No idea wht could cause this -- just raise it to the caller. + goto error_return; + } + + if ((value < 0) || (value > SEMAPHORE_VALUE_MAX)) { + PyErr_Format(PyExc_ValueError, + "Attribute 'value' must be between 0 and %ld (SEMAPHORE_VALUE_MAX)", + (long)SEMAPHORE_VALUE_MAX); + goto error_return; + } + + arg.val = value; + + if (-1 == semctl(self->id, 0, SETVAL, arg)) { + sem_set_error(); + goto error_return; + } + + return 0; + + error_return: + return -1; +} + + +PyObject * +sem_get_block(Semaphore *self) { + DPRINTF("op_flags: %x\n", self->op_flags); + return PyBool_FromLong( (self->op_flags & IPC_NOWAIT) ? 0 : 1); +} + + +int +sem_set_block(Semaphore *self, PyObject *py_value) +{ + DPRINTF("op_flags before: %x\n", self->op_flags); + + if (PyObject_IsTrue(py_value)) + self->op_flags &= ~IPC_NOWAIT; + else + self->op_flags |= IPC_NOWAIT; + + DPRINTF("op_flags after: %x\n", self->op_flags); + + return 0; +} + + +PyObject * +sem_get_mode(Semaphore *self) { + return sem_get_ipc_perm_value(self->id, SVIFP_IPC_PERM_MODE); +} + + +int +sem_set_mode(Semaphore *self, PyObject *py_value) { + return sem_set_ipc_perm_value(self->id, SVIFP_IPC_PERM_MODE, py_value); +} + + +PyObject * +sem_get_undo(Semaphore *self) { + return PyBool_FromLong( (self->op_flags & SEM_UNDO) ? 1 : 0 ); +} + + +int +sem_set_undo(Semaphore *self, PyObject *py_value) +{ + DPRINTF("op_flags before: %x\n", self->op_flags); + + if (PyObject_IsTrue(py_value)) + self->op_flags |= SEM_UNDO; + else + self->op_flags &= ~SEM_UNDO; + + DPRINTF("op_flags after: %x\n", self->op_flags); + + return 0; +} + + +PyObject * +sem_get_uid(Semaphore *self) { + return sem_get_ipc_perm_value(self->id, SVIFP_IPC_PERM_UID); +} + + +int +sem_set_uid(Semaphore *self, PyObject *py_value) { + return sem_set_ipc_perm_value(self->id, SVIFP_IPC_PERM_UID, py_value); +} + + +PyObject * +sem_get_gid(Semaphore *self) { + return sem_get_ipc_perm_value(self->id, SVIFP_IPC_PERM_GID); +} + + +int +sem_set_gid(Semaphore *self, PyObject *py_value) { + return sem_set_ipc_perm_value(self->id, SVIFP_IPC_PERM_GID, py_value); +} + +PyObject * +sem_get_c_uid(Semaphore *self) { + return sem_get_ipc_perm_value(self->id, SVIFP_IPC_PERM_CUID); +} + +PyObject * +sem_get_c_gid(Semaphore *self) { + return sem_get_ipc_perm_value(self->id, SVIFP_IPC_PERM_CGID); +} + +PyObject * +sem_get_last_pid(Semaphore *self) { + return sem_get_semctl_value(self->id, GETPID); +} + +PyObject * +sem_get_waiting_for_nonzero(Semaphore *self) { + return sem_get_semctl_value(self->id, GETNCNT); +} + +PyObject * +sem_get_waiting_for_zero(Semaphore *self) { + return sem_get_semctl_value(self->id, GETZCNT); +} + +PyObject * +sem_get_o_time(Semaphore *self) { + return sem_get_ipc_perm_value(self->id, SVIFP_SEM_OTIME); +} + diff --git a/semaphore.h b/semaphore.h new file mode 100644 index 0000000..b4ee63d --- /dev/null +++ b/semaphore.h @@ -0,0 +1,53 @@ +typedef struct { + PyObject_HEAD + key_t key; + int id; + short op_flags; +} Semaphore; + + +/* Object methods */ +PyObject *Semaphore_new(PyTypeObject *type, PyObject *, PyObject *); +int Semaphore_init(Semaphore *, PyObject *, PyObject *); +void Semaphore_dealloc(Semaphore *); +PyObject *Semaphore_enter(Semaphore *); +PyObject *Semaphore_exit(Semaphore *, PyObject *); +PyObject *Semaphore_P(Semaphore *, PyObject *, PyObject *); +PyObject *Semaphore_acquire(Semaphore *, PyObject *, PyObject *); +PyObject *Semaphore_V(Semaphore *, PyObject *, PyObject *); +PyObject *Semaphore_release(Semaphore *, PyObject *, PyObject *); +PyObject *Semaphore_Z(Semaphore *, PyObject *, PyObject *); +PyObject *Semaphore_remove(Semaphore *); + +/* Object attributes (read-write & read-only) */ +PyObject *sem_get_value(Semaphore *); +int sem_set_value(Semaphore *self, PyObject *py_value); + +PyObject *sem_get_block(Semaphore *); +int sem_set_block(Semaphore *self, PyObject *py_value); + +PyObject *sem_get_mode(Semaphore *); +int sem_set_mode(Semaphore *, PyObject *); + +PyObject *sem_get_undo(Semaphore *); +int sem_set_undo(Semaphore *self, PyObject *py_value); + +PyObject *sem_get_uid(Semaphore *); +int sem_set_uid(Semaphore *, PyObject *); + +PyObject *sem_get_gid(Semaphore *); +int sem_set_gid(Semaphore *, PyObject *); + +PyObject *sem_get_key(Semaphore *); +PyObject *sem_get_c_uid(Semaphore *); +PyObject *sem_get_c_gid(Semaphore *); +PyObject *sem_get_last_pid(Semaphore *); +PyObject *sem_get_waiting_for_nonzero(Semaphore *); +PyObject *sem_get_waiting_for_zero(Semaphore *); +PyObject *sem_get_o_time(Semaphore *); + +PyObject *sem_str(Semaphore *); +PyObject *sem_repr(Semaphore *); + +/* Utility functions */ +PyObject *sem_remove(int); diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f1dfc2f --- /dev/null +++ b/setup.py @@ -0,0 +1,62 @@ +# Python modules +import distutils.core as duc + +# sysv_ipc installation helper module +import prober + +VERSION = open("VERSION", "r").read().strip() + +name = "sysv_ipc" +description = "System V IPC primitives (semaphores, shared memory and message queues) for Python" +long_description = open("README", "r").read() +author = "Philip Semanchuk" +author_email = "philip@semanchuk.com" +maintainer = "Philip Semanchuk" +url = "http://semanchuk.com/philip/sysv_ipc/" +download_url = "http://semanchuk.com/philip/sysv_ipc/sysv_ipc-%s.tar.gz" % VERSION +source_files = ["sysv_ipc_module.c", "common.c", "semaphore.c", "memory.c", + "mq.c" ] +# http://pypi.python.org/pypi?:action=list_classifiers +classifiers = [ "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: BSD :: FreeBSD", + "Operating System :: POSIX :: Linux", + "Operating System :: POSIX :: SunOS/Solaris", + "Operating System :: POSIX", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Topic :: Utilities" ] +license = "http://creativecommons.org/licenses/BSD/" +keywords = "ipc inter-process communication semaphore shared memory shm message queue" + +prober.probe() + +extension = duc.Extension("sysv_ipc", + source_files, +# extra_compile_args=['-E'] + depends = [ "common.c", "common.h", "memory.c", + "memory.h", "mq.c", "mq.h", + "probe_results.h", "semaphore.c", + "semaphore.h", "sysv_ipc_module.c", + ], + ) + +duc.setup(name = name, + version = VERSION, + description = description, + long_description = long_description, + author = author, + author_email = author_email, + maintainer = maintainer, + url = url, + download_url = download_url, + classifiers = classifiers, + license = license, + keywords = keywords, + ext_modules = [ extension ] + ) + diff --git a/sysv_ipc_module.c b/sysv_ipc_module.c new file mode 100644 index 0000000..fd0dfdf --- /dev/null +++ b/sysv_ipc_module.c @@ -0,0 +1,876 @@ +/* +sysv_ipc - A Python module for accessing System V semaphores, shared memory + and message queues. + +Copyright (c) 2008, Philip Semanchuk +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of sysv_ipc nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY Philip Semanchuk ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL Philip Semanchuk BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +#include "Python.h" +#include "structmember.h" + +// For memset +#include + +// For srand +#include +#include + +// For the math surrounding timeouts for semtimedop() +#include + +#include "common.h" +#include "semaphore.h" +#include "memory.h" +#include "mq.h" + +PyObject *pBaseException; +PyObject *pInternalException; +PyObject *pPermissionsException; +PyObject *pExistentialException; +PyObject *pBusyException; +PyObject *pNotAttachedException; + +// sysv_ipc_attach() needs this forward declaration of SharedMemoryType +static PyTypeObject SharedMemoryType; + +/* + + Module methods + +*/ + + +static PyObject * +sysv_ipc_attach(PyObject *self, PyObject *args, PyObject *keywords) { + // Given the id of an extant shared memory segment and (optionally) an + // address and shmat flags, attempts to attach the memory. If successful, + // returns a new SharedMemory object with the key permanently set to -1. + SharedMemory *shm = NULL; + PyObject *py_address = NULL; + int id = -1; + void *address = NULL; + int flags = 0; + char *keyword_list[ ] = {"id", "address", "flags", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, keywords, "i|Oi", keyword_list, + &id, &py_address, &flags)) + goto error_return; + + if ((!py_address) || (py_address == Py_None)) + address = NULL; + else { + if (PyLong_Check(py_address)) + address = PyLong_AsVoidPtr(py_address); + else { + PyErr_SetString(PyExc_TypeError, "address must be a long"); + goto error_return; + } + } + + DPRINTF("About to create a new SharedMemory object.\n"); + + /* Create a new SharedMemory object. Some tutorials recommend using + PyObject_CallObject() to create this, but that invokes the __init__ method + which I don't want to do. + */ + shm = (SharedMemory *)PyObject_New(SharedMemory, &SharedMemoryType); + shm->id = id; + shm->address = address; + + DPRINTF("About to call shm_attach()\n"); + if (Py_None == shm_attach(shm, flags)) + // All is well + return (PyObject *)shm; + else + // abandon this object and fall through to the error return below. + Py_DECREF(shm); + + error_return: + return NULL; +} + + +static PyObject * +sysv_ipc_ftok(PyObject *self, PyObject *args, PyObject *keywords) { + char *path; + int id = 0; + int silence_warning = 0; + char *keyword_list[ ] = {"path", "id", "silence_warning", NULL}; + + key_t rc = 0; + + if (!PyArg_ParseTupleAndKeywords(args, keywords, "si|i", keyword_list, + &path, &id, &silence_warning)) + goto error_return; + + if (!silence_warning) { + DPRINTF("path=%s, id=%d, rc=%ld\n", path, id, rc); + PyErr_WarnEx(PyExc_Warning, + "Use of ftok() is not recommended; see sysv_ipc documentation", 1); + } + + rc = ftok(path, id); + + DPRINTF("path=%s, id=%d, rc=%ld\n", path, id, rc); + + return Py_BuildValue("i", rc); + + error_return: + return NULL; +} + + +static PyObject * +sysv_ipc_remove_semaphore(PyObject *self, PyObject *args) { + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + goto error_return; + + DPRINTF("removing sem with id %d\n", id); + if (NULL == sem_remove(id)) + goto error_return; + + Py_RETURN_NONE; + + error_return: + return NULL; +} + + +static PyObject * +sysv_ipc_remove_shared_memory(PyObject *self, PyObject *args) { + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + goto error_return; + + return shm_remove(id); + + error_return: + return NULL; +} + + +static PyObject * +sysv_ipc_remove_message_queue(PyObject *self, PyObject *args) { + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + goto error_return; + + return mq_remove(id); + + error_return: + return NULL; +} + + +static PyMemberDef Semaphore_members[] = { + {"id", T_INT, offsetof(Semaphore, id), READONLY, "The id assigned by the system"}, + {NULL} /* Sentinel */ +}; + + +static PyMethodDef Semaphore_methods[] = { + { "__enter__", + (PyCFunction)Semaphore_enter, + METH_NOARGS, + }, + { "__exit__", + (PyCFunction)Semaphore_exit, + METH_VARARGS, + }, + { "P", + (PyCFunction)Semaphore_P, + METH_VARARGS | METH_KEYWORDS, + "Acquire (decrement) the semaphore, waiting if necessary" + }, + { "acquire", + (PyCFunction)Semaphore_acquire, + METH_VARARGS | METH_KEYWORDS, + "Acquire (decrement) the semaphore, waiting if necessary" + }, + { "V", + (PyCFunction)Semaphore_V, + METH_VARARGS | METH_KEYWORDS, + "Release (increment) the semaphore" + }, + { "release", + (PyCFunction)Semaphore_release, + METH_VARARGS | METH_KEYWORDS, + "Release (increment) the semaphore" + }, + { "Z", + (PyCFunction)Semaphore_Z, + METH_VARARGS | METH_KEYWORDS, + "Waits until zee zemaphore is zero" + }, + { "remove", + (PyCFunction)Semaphore_remove, + METH_NOARGS, + "Removes (deletes) the semaphore from the system" + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + + +static PyGetSetDef Semaphore_gets_and_sets[] = { + { "key", + (getter)sem_get_key, + (setter)NULL, + "The key passed to the constructor", + NULL + }, + { "value", + (getter)sem_get_value, + (setter)sem_set_value, + "The semaphore's current value", + NULL + }, + { "undo", + (getter)sem_get_undo, + (setter)sem_set_undo, + "When True, acquire/release operations will be undone when the process exits. Non-portable.", + NULL + }, + { "block", + (getter)sem_get_block, + (setter)sem_set_block, + "When True (the default), calls to acquire/release/P/V/Z will wait (block) if the semaphore is busy", + NULL + }, + { "mode", + (getter)sem_get_mode, + (setter)sem_set_mode, + "Permissions", + NULL + }, + { "uid", + (getter)sem_get_uid, + (setter)sem_set_uid, + "The semaphore's UID", + NULL + }, + { "gid", + (getter)sem_get_gid, + (setter)sem_set_gid, + "The semaphore's GID", + NULL + }, + { "cuid", + (getter)sem_get_c_uid, + (setter)NULL, + "The semaphore creator's UID. Read only.", + NULL + }, + { "cgid", + (getter)sem_get_c_gid, + (setter)NULL, + "The semaphore creator's GID. Read only.", + NULL + }, + { "last_pid", + (getter)sem_get_last_pid, + (setter)NULL, + "The id of the last process to call acquire()/release()/Z() on this semaphore. Read only.", + NULL + }, + { "waiting_for_nonzero", + (getter)sem_get_waiting_for_nonzero, + (setter)NULL, + "The number of processes waiting for the semaphore to become non-zero. Read only.", + NULL + }, + { "waiting_for_zero", + (getter)sem_get_waiting_for_zero, + (setter)NULL, + "The number of processes waiting for the semaphore to become zero. Read only.", + NULL + }, + { "o_time", + (getter)sem_get_o_time, + (setter)NULL, + "The last time semop (acquire/release/P/V/Z) was called on this semaphore. Initialized to zero. Read only.", + NULL + }, + {NULL} /* Sentinel */ +}; + + + + +static PyTypeObject SemaphoreType = { + PyVarObject_HEAD_INIT(NULL, 0) + "sysv_ipc.Semaphore", // tp_name + sizeof(Semaphore), // tp_basicsize + 0, // tp_itemsize + (destructor)Semaphore_dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + (reprfunc)sem_repr, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + (reprfunc)sem_str, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags + "System V semaphore object", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + Semaphore_methods, // tp_methods + Semaphore_members, // tp_members + Semaphore_gets_and_sets, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + (initproc)Semaphore_init, // tp_init + 0, // tp_alloc + Semaphore_new, // tp_new +}; + + + +static PyMemberDef SharedMemory_members[] = { + {"id", T_INT, offsetof(SharedMemory, id), READONLY, "The id assigned by the system"}, + {NULL} /* Sentinel */ +}; + + +static PyMethodDef SharedMemory_methods[] = { + { "read", + (PyCFunction)SharedMemory_read, + METH_VARARGS | METH_KEYWORDS, + "Read n bytes from the shared memory at the given offset into a Python string" + }, + { "write", + (PyCFunction)SharedMemory_write, + METH_VARARGS | METH_KEYWORDS, + "Write the string to the shared memory at the offset given" + }, + { "remove", + (PyCFunction)SharedMemory_remove, + METH_NOARGS, + "Removes (deletes) the shared memory from the system" + }, + { "attach", + (PyCFunction)SharedMemory_attach, + METH_VARARGS, + "Attaches the shared memory" + }, + { "detach", + (PyCFunction)SharedMemory_detach, + METH_NOARGS, + "Detaches the shared memory" + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + + +static PyGetSetDef SharedMemory_gets_and_sets[] = { + { "key", + (getter)shm_get_key, + (setter)NULL, + "The key passed to the constructor. Read only.", + NULL + }, + { "size", + (getter)shm_get_size, + (setter)NULL, + "The size of the segment in bytes. Read only.", + NULL + }, + { "address", + (getter)shm_get_address, + (setter)NULL, + "The memory address of the segment. Read only.", + NULL + }, + { "attached", + (getter)shm_get_attached, + (setter)NULL, + "True if the segment is attached. Read only.", + NULL + }, + { "last_attach_time", + (getter)shm_get_last_attach_time, + (setter)NULL, + "The most recent time this segment was attached. Read only.", + NULL + }, + { "last_detach_time", + (getter)shm_get_last_detach_time, + (setter)NULL, + "The most recent time this segment was detached. Read only.", + NULL + }, + { "last_change_time", + (getter)shm_get_last_change_time, + (setter)NULL, + "The time of the most recent change to this segment's uid, gid, mode, or the time the segment was removed. Read only.", + NULL + }, + { "creator_pid", + (getter)shm_get_creator_pid, + (setter)NULL, + "The process id of the creator. Read only.", + NULL + }, + { "last_pid", + (getter)shm_get_last_pid, + (setter)NULL, + "The id of the process that performed the most recent attach or detach. Read only.", + NULL + }, + { "number_attached", + (getter)shm_get_number_attached, + (setter)NULL, + "The current number of attached processes. Read only.", + NULL + }, + { "uid", + (getter)shm_get_uid, + (setter)shm_set_uid, + "The segment's UID.", + NULL + }, + { "gid", + (getter)shm_get_gid, + (setter)shm_set_gid, + "The segment's GID.", + NULL + }, + { "cuid", + (getter)shm_get_cuid, + (setter)NULL, + "The UID of the segment's creator. Read only.", + NULL + }, + { "cgid", + (getter)shm_get_cgid, + (setter)NULL, + "The GID of the segment's creator. Read only.", + NULL + }, + { "mode", + (getter)shm_get_mode, + (setter)shm_set_mode, + "Permissions.", + NULL + }, + {NULL} /* Sentinel */ +}; + + + +static PyTypeObject SharedMemoryType = { + PyVarObject_HEAD_INIT(NULL, 0) + "sysv_ipc.SharedMemory", // tp_name + sizeof(SharedMemory), // tp_basicsize + 0, // tp_itemsize + (destructor)SharedMemory_dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + (reprfunc)shm_repr, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + (reprfunc)shm_str, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags + "System V shared memory object", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + SharedMemory_methods, // tp_methods + SharedMemory_members, // tp_members + SharedMemory_gets_and_sets, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + (initproc)SharedMemory_init, // tp_init + 0, // tp_alloc + SharedMemory_new, // tp_new +}; + + + + +static PyMemberDef MessageQueue_members[] = { + {"id", T_INT, offsetof(MessageQueue, id), READONLY, "Message queue id"}, + {NULL} /* Sentinel */ +}; + + +static PyMethodDef MessageQueue_methods[] = { + { "send", + (PyCFunction)MessageQueue_send, + METH_VARARGS | METH_KEYWORDS, + "Place a message on the queue" + }, + { "receive", + (PyCFunction)MessageQueue_receive, + METH_VARARGS | METH_KEYWORDS, + "Receive a message from the queue" + }, + { "remove", + (PyCFunction)MessageQueue_remove, + METH_NOARGS, + "Removes (deletes) the queue from the system" + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + + +static PyGetSetDef MessageQueue_gets_and_sets[] = { + { "key", + (getter)mq_get_key, + (setter)NULL, + "The key passed to the constructor.", + NULL + }, + { "last_send_time", + (getter)mq_get_last_send_time, + (setter)NULL, + "A Unix timestamp representing the last time a message was sent.", + NULL + }, + { "last_receive_time", + (getter)mq_get_last_receive_time, + (setter)NULL, + "A Unix timestamp representing the last time a message was received.", + NULL + }, + { "last_change_time", + (getter)mq_get_last_change_time, + (setter)NULL, + "A Unix timestamp representing the last time the queue was changed.", + NULL + }, + { "current_messages", + (getter)mq_get_current_messages, + (setter)NULL, + "The number of messages currently in the queue", + NULL + }, + { "last_send_pid", + (getter)mq_get_last_send_pid, + (setter)NULL, + "The id of the last process which sent via the queue", + NULL}, + { "last_receive_pid", + (getter)mq_get_last_receive_pid, + (setter)NULL, + "The id of the last process which received from the queue", + NULL + }, + { "max_size", + (getter)mq_get_max_size, + (setter)mq_set_max_size, + "The maximum size of the queue (in bytes). Read-write if you have sufficient privileges.", + NULL + }, + { "mode", + (getter)mq_get_mode, + (setter)mq_set_mode, + "Permissions", + NULL + }, + { "uid", + (getter)mq_get_uid, + (setter)mq_set_uid, + "The queue's UID.", + NULL + }, + { "gid", + (getter)mq_get_gid, + (setter)mq_set_gid, + "The queue's GID.", + NULL + }, + { "cuid", + (getter)mq_get_c_uid, + (setter)NULL, + "The UID of the queue's creator. Read only.", + NULL + }, + { "cgid", + (getter)mq_get_c_gid, + (setter)NULL, + "The GID of the queue's creator. Read only.", + NULL + }, + {NULL} /* Sentinel */ +}; + + +static PyTypeObject MessageQueueType = { + PyVarObject_HEAD_INIT(NULL, 0) + "sysv_ipc.MessageQueue", // tp_name + sizeof(MessageQueue), // tp_basicsize + 0, // tp_itemsize + (destructor)MessageQueue_dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + (reprfunc)mq_repr, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + (reprfunc)mq_str, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags + "System V message queue object", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + MessageQueue_methods, // tp_methods + MessageQueue_members, // tp_members + MessageQueue_gets_and_sets, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + (initproc)MessageQueue_init, // tp_init + 0, // tp_alloc + MessageQueue_new, // tp_new +}; + + +/* + + Module level stuff + +*/ + +static PyMethodDef module_methods[ ] = { + { "attach", + (PyCFunction)sysv_ipc_attach, + METH_VARARGS | METH_KEYWORDS, + "Attaches the memory identified by the id and returns a new SharedMemory object." + }, + { "ftok", + (PyCFunction)sysv_ipc_ftok, + METH_VARARGS | METH_KEYWORDS, + "Calls ftok(). Not recommended; see sysv_ipc documentation." + }, + { "remove_semaphore", + (PyCFunction)sysv_ipc_remove_semaphore, + METH_VARARGS, + "Remove (delete) the semaphore identified by id" + }, + { "remove_shared_memory", + (PyCFunction)sysv_ipc_remove_shared_memory, + METH_VARARGS, + "Remove shared memory identified by id" + }, + { "remove_message_queue", + (PyCFunction)sysv_ipc_remove_message_queue, + METH_VARARGS, + "Remove the message queue identified by id" + }, + {NULL} /* Sentinel */ +}; + + +#if PY_MAJOR_VERSION > 2 +static struct PyModuleDef this_module = { + PyModuleDef_HEAD_INIT, // m_base + "sysv_ipc", // m_name + "SYSV IPC module", // m_doc + -1, // m_size (space allocated for module globals) + module_methods, // m_methods + NULL, // m_reload + NULL, // m_traverse + NULL, // m_clear + NULL // m_free +}; +#endif + + +/* Module init function */ +#if PY_MAJOR_VERSION > 2 +#define SYSV_IPC_INIT_FUNCTION_NAME PyInit_sysv_ipc +#else +#define SYSV_IPC_INIT_FUNCTION_NAME initsysv_ipc +#endif + + +/* Module init function */ +PyMODINIT_FUNC +SYSV_IPC_INIT_FUNCTION_NAME(void) { + PyObject *module; + PyObject *module_dict; + + // I seed the random number generator in case I'm asked to make some + // random keys. + srand((unsigned int)time(NULL)); + +#if PY_MAJOR_VERSION > 2 + module = PyModule_Create(&this_module); +#else + module = Py_InitModule3("sysv_ipc", module_methods, "System V IPC module"); +#endif + + if (!module) + goto error_return; + + if (PyType_Ready(&SemaphoreType) < 0) + goto error_return; + + if (PyType_Ready(&SharedMemoryType) < 0) + goto error_return; + + if (PyType_Ready(&MessageQueueType) < 0) + goto error_return; + +#ifdef SEMTIMEDOP_EXISTS + Py_INCREF(Py_True); + PyModule_AddObject(module, "SEMAPHORE_TIMEOUT_SUPPORTED", Py_True); +#else + Py_INCREF(Py_False); + PyModule_AddObject(module, "SEMAPHORE_TIMEOUT_SUPPORTED", Py_False); +#endif + + PyModule_AddStringConstant(module, "VERSION", SYSV_IPC_VERSION); + PyModule_AddStringConstant(module, "__version__", SYSV_IPC_VERSION); + PyModule_AddStringConstant(module, "__copyright__", "Copyright 2008 - 2014 Philip Semanchuk"); + PyModule_AddStringConstant(module, "__author__", "Philip Semanchuk"); + PyModule_AddStringConstant(module, "__license__", "BSD"); + + PyModule_AddIntConstant(module, "PAGE_SIZE", PAGE_SIZE); + PyModule_AddIntConstant(module, "KEY_MIN", KEY_MIN); + PyModule_AddIntConstant(module, "KEY_MAX", KEY_MAX); + PyModule_AddIntConstant(module, "SEMAPHORE_VALUE_MAX", SEMAPHORE_VALUE_MAX); + PyModule_AddIntConstant(module, "IPC_CREAT", IPC_CREAT); + PyModule_AddIntConstant(module, "IPC_EXCL", IPC_EXCL); + PyModule_AddIntConstant(module, "IPC_CREX", IPC_CREX); + PyModule_AddIntConstant(module, "IPC_PRIVATE", IPC_PRIVATE); + PyModule_AddIntConstant(module, "SHM_RND", SHM_RND); + PyModule_AddIntConstant(module, "SHM_RDONLY", SHM_RDONLY); + + + // These flags are Linux-specific. +#ifdef SHM_HUGETLB + PyModule_AddIntConstant(module, "SHM_HUGETLB", SHM_HUGETLB); +#endif +#ifdef SHM_NORESERVE + PyModule_AddIntConstant(module, "SHM_NORESERVE", SHM_NORESERVE); +#endif +#ifdef SHM_REMAP + PyModule_AddIntConstant(module, "SHM_REMAP", SHM_REMAP); +#endif + + Py_INCREF(&SemaphoreType); + PyModule_AddObject(module, "Semaphore", (PyObject *)&SemaphoreType); + + Py_INCREF(&SharedMemoryType); + PyModule_AddObject(module, "SharedMemory", (PyObject *)&SharedMemoryType); + + Py_INCREF(&MessageQueueType); + PyModule_AddObject(module, "MessageQueue", (PyObject *)&MessageQueueType); + + // Exceptions + if (!(module_dict = PyModule_GetDict(module))) + goto error_return; + + if (!(pBaseException = PyErr_NewException("sysv_ipc.Error", NULL, NULL))) + goto error_return; + else + PyDict_SetItemString(module_dict, "Error", pBaseException); + + if (!(pInternalException = PyErr_NewException("sysv_ipc.InternalError", NULL, NULL))) + goto error_return; + else + PyDict_SetItemString(module_dict, "InternalError", pInternalException); + + if (!(pPermissionsException = PyErr_NewException("sysv_ipc.PermissionsError", pBaseException, NULL))) + goto error_return; + else + PyDict_SetItemString(module_dict, "PermissionsError", pPermissionsException); + + if (!(pExistentialException = PyErr_NewException("sysv_ipc.ExistentialError", pBaseException, NULL))) + goto error_return; + else + PyDict_SetItemString(module_dict, "ExistentialError", pExistentialException); + + if (!(pBusyException = PyErr_NewException("sysv_ipc.BusyError", pBaseException, NULL))) + goto error_return; + else + PyDict_SetItemString(module_dict, "BusyError", pBusyException); + + if (!(pNotAttachedException = PyErr_NewException("sysv_ipc.NotAttachedError", pBaseException, NULL))) + goto error_return; + else + PyDict_SetItemString(module_dict, "NotAttachedError", pNotAttachedException); + +#if PY_MAJOR_VERSION > 2 + return module; +#endif + + error_return: +#if PY_MAJOR_VERSION > 2 + return NULL; +#else + ; // Nothing to do +#endif +} + -- cgit v1.2.1