PyMeta2 Example

I found it difficult to find examples on the use of PyMeta2, an implememtation of the OMeta language in Python. I mamaged to put together a small example myself on how to parse a PGP public key block and extract the DSA key.

pgp.py:

from pymeta.grammar import OMeta

# Base 64 decoder, see RFC 2045.
grammar = """
code    = letter:x ?(x <= 'Z') -> (ord(x) - ord('A'))
        | letter:x ?(x <= 'z') -> (ord(x) - ord('a') + 26)
        | digit:y              -> (ord(y) - ord('0') + 52)
        | '+'                  -> (62)
        | '/'                  -> (63)
sixbits = spaces code:z -> (z)
        | code:z        -> (z)
pad     = spaces '='
        | '='
group = sixbits:a sixbits:b sixbits:c sixbits:d
        -> ([(a<<2)+(b>>4), ((b&0xf)<<4)+(c>>2), ((c&0x3)<<6)+d]) 
      | sixbits:a sixbits:b sixbits:c pad
        -> ([(a<<2)+(b>>4), ((b&0xf)<<4)+(c>>2)])
      | sixbits:a sixbits:b pad pad
        -> ([(a<<2)+(b>>4)])
data = data:s group:z -> (s + z)
     | group:z        -> (z)
packet = data:d spaces '=' data:c spaces end -> ({'data': d, 'checksum': c})
"""

parser = OMeta.makeGrammar(grammar, {})

# Data from PGP PUBLIC KEY BLOCK
code = """
mQGiBE5cxWkRBACgm0DoNQj5HTTcQLL/eooKYMJTmYiINk0bSfQ46+R1iQliL/67
K0MTUZltFv3z70ATY9HulMgmlutDsFRGG4yUWceQX0qNBbJv2Q7FzB2rAFd/XfHT
tB8EOYc1fZk2RCDk0m5Ef33kwVXMm7/+MyvJ4GWxd8rvEsvrLdVZGufZ1wCg+K9p
Qgm542nvRI8LZN91yZGEiMcD/Rv6+oYykhOfU4ozxpOv2CLjzF3qk+D8TvVFEDrE
yjfoZZBGBHWY8bWyMsWONjk3o2DFSwQ5qwhz/bgcUtTPj8qU9K8KyrLB1mmiRTj1
+6svwUORoOCqKmTXkna7bmzzdlsqRQR2g9AdMHuIDkynyE7PBEYO6r7roUZZVtHU
TIRiBACRiV9ezsDsKRNfFBvoI5Lip7aA81E3+aesDGvGLqCeje3NIJfZ5q0q02iE
WTkNduAwxjcP+Q6QnakCoTSK7tlyiFWTw+bc0fq4V72hey+PqBclk8gRFDoCSuCF
vJOty98GfATei8fBJIpL4+hLNSGbN6QIKWtakS3IorWVXfeqHrQHVGFjdGl2b4hg
BBMRAgAgBQJOXMVpAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQ9t723H2z
gCa0hgCcCH6NcL6fBrf3uHckMAFvPLub0SYAn2HeLhGECfI8pJ06S7aT/oPX0tyU
uQENBE5cxWkQBACGpC50KVpoy+4QmyIUHtSVQ4gbfeGOMmHOZlRRs41t93iqdf8F
y0aDUiK1G4AbnpHgJL/IFfH2HUPn1v7A4zRP9kd14k7+V5hjtiFj6m+USz/9M2Cd
CCWC0b7JAwK0egCDoGfdDPZBXF73ec5ObntinBg+YqBy7GDMCsFgZ8fjKwADBgP9
HUCsOp+BikAKHErqi5p73vRkzGq4kLs32ixwWP/6F2wcR2mJlgR7ET0gopvlkx9A
fDptYO4TG0nNVtBF/GqEbUIBcEfvpDVtVgstW25Aj367x06ySOWD1A/7No3j2YY8
bXPRxTDmUpyJNhcFDauaSClffI75UbYTMreuG9JbiYuISQQYEQIACQUCTlzFaQIb
DAAKCRD23vbcfbOAJqBvAKDocpghK+6fgmDiz6NDnWsnjKHOLgCfVuO99xoUFd+b
IWBs+tbQvkoRK8U=
=9tpq
"""

packet = parser(code).apply("packet")


# Checksum calculator
def crc24(octets):
    INIT = 0xB704CE
    POLY = 0x1864CFB
    crc = INIT
    for octet in octets:
        crc ^= (octet << 16)
        for i in xrange(8):
            crc <<= 1
            if crc & 0x1000000: crc ^= POLY
    return crc & 0xFFFFFF

# Compare values in checksum manually
checksum = list()

# Checksum in code
checksum.append(packet[0]['checksum'][0]*256*256 + packet[0]['checksum'][1]*256 + packet[0]['checksum'][2])

# Calculated checksum
checksum.append(crc24(packet[0]['data']))


# DSA public-key packet parser, see RFC 4880 
blockgrammar = """
number 0 =                               -> (0)
number :m = number(m-1):msb anything:lsb -> (msb*256+lsb)

length 0 = number(1):l -> (l)
length 1 = number(2):l -> (l)
length 2 = number(4):l -> (l)

mpi = number(2):bits number((bits+7)/8):i -> ([bits, i])

block = anything:tag length(tag&0x03):l anything:ver number(4):t anything:algo mpi:p mpi:q mpi:g mpi:y
        -> ({'tag': (tag&0x7f)>>2, 'length': l, 'version': ver, 'time': t, 'algorithm': algo, 'p': p, 'q': q, 'g': g, 'y': y})
"""

blockparser = OMeta.makeGrammar(blockgrammar, {})

block = blockparser(packet[0]['data']).apply("block")

A session using this code would look something like this:

Python 2.7.1 (r271:86832, Nov 27 2010, 18:30:46) [MSC v.1500 32 bit (Intel)] on
win32
Type "help", "copyright", "credits" or "license" for more information.
>>> import pgp
>>> pgp.packet
({'checksum': [246, 218, 106], 'data': [153, 1, 162, 4, 78, 92, 197, 105, 17, 4,
 0, 160, 155, 64, 232, 53, 8, 249, 29, 52, 220, 64, 178, 255, 122, 138, 10, 96,
194, 83, 153, 136, 136, 54, 77, 27, 73, 244, 56, 235, 228, 117, 137, 9, 98, 47,
254, 187, 43, 67, 19, 81, 153, 109, 22, 253, 243, 239, 64, 19, 99, 209, 238, 148
, 200, 38, 150, 235, 67, 176, 84, 70, 27, 140, 148, 89, 199, 144, 95, 74, 141, 5
, 178, 111, 217, 14, 197, 204, 29, 171, 0, 87, 127, 93, 241, 211, 180, 31, 4, 57
, 135, 53, 125, 153, 54, 68, 32, 228, 210, 110, 68, 127, 125, 228, 193, 85, 204,
 155, 191, 254, 51, 43, 201, 224, 101, 177, 119, 202, 239, 18, 203, 235, 45, 213
, 89, 26, 231, 217, 215, 0, 160, 248, 175, 105, 66, 9, 185, 227, 105, 239, 68, 1
43, 11, 100, 223, 117, 201, 145, 132, 136, 199, 3, 253, 27, 250, 250, 134, 50, 1
46, 19, 159, 83, 138, 51, 198, 147, 175, 216, 34, 227, 204, 93, 234, 147, 224, 2
52, 78, 245, 69, 16, 58, 196, 202, 55, 232, 101, 144, 70, 4, 117, 152, 241, 181,
 178, 50, 197, 142, 54, 57, 55, 163, 96, 197, 75, 4, 57, 171, 8, 115, 253, 184,
28, 82, 212, 207, 143, 202, 148, 244, 175, 10, 202, 178, 193, 214, 105, 162, 69,
 56, 245, 251, 171, 47, 193, 67, 145, 160, 224, 170, 42, 100, 215, 146, 118, 187
, 110, 108, 243, 118, 91, 42, 69, 4, 118, 131, 208, 29, 48, 123, 136, 14, 76, 16
7, 200, 78, 207, 4, 70, 14, 234, 190, 235, 161, 70, 89, 86, 209, 212, 76, 132, 9
8, 4, 0, 145, 137, 95, 94, 206, 192, 236, 41, 19, 95, 20, 27, 232, 35, 146, 226,
 167, 182, 128, 243, 81, 55, 249, 167, 172, 12, 107, 198, 46, 160, 158, 141, 237
, 205, 32, 151, 217, 230, 173, 42, 211, 104, 132, 89, 57, 13, 118, 224, 48, 198,
 55, 15, 249, 14, 144, 157, 169, 2, 161, 52, 138, 238, 217, 114, 136, 85, 147, 1
95, 230, 220, 209, 250, 184, 87, 189, 161, 123, 47, 143, 168, 23, 37, 147, 200,
17, 20, 58, 2, 74, 224, 133, 188, 147, 173, 203, 223, 6, 124, 4, 222, 139, 199,
193, 36, 138, 75, 227, 232, 75, 53, 33, 155, 55, 164, 8, 41, 107, 90, 145, 45, 2
00, 162, 181, 149, 93, 247, 170, 30, 180, 7, 84, 97, 99, 116, 105, 118, 111, 136
, 96, 4, 19, 17, 2, 0, 32, 5, 2, 78, 92, 197, 105, 2, 27, 3, 6, 11, 9, 8, 7, 3,
2, 4, 21, 2, 8, 3, 4, 22, 2, 3, 1, 2, 30, 1, 2, 23, 128, 0, 10, 9, 16, 246, 222,
 246, 220, 125, 179, 128, 38, 180, 134, 0, 156, 8, 126, 141, 112, 190, 159, 6, 1
83, 247, 184, 119, 36, 48, 1, 111, 60, 187, 155, 209, 38, 0, 159, 97, 222, 46, 1
7, 132, 9, 242, 60, 164, 157, 58, 75, 182, 147, 254, 131, 215, 210, 220, 148, 18
5, 1, 13, 4, 78, 92, 197, 105, 16, 4, 0, 134, 164, 46, 116, 41, 90, 104, 203, 23
8, 16, 155, 34, 20, 30, 212, 149, 67, 136, 27, 125, 225, 142, 50, 97, 206, 102,
84, 81, 179, 141, 109, 247, 120, 170, 117, 255, 5, 203, 70, 131, 82, 34, 181, 27
, 128, 27, 158, 145, 224, 36, 191, 200, 21, 241, 246, 29, 67, 231, 214, 254, 192
, 227, 52, 79, 246, 71, 117, 226, 78, 254, 87, 152, 99, 182, 33, 99, 234, 111, 1
48, 75, 63, 253, 51, 96, 157, 8, 37, 130, 209, 190, 201, 3, 2, 180, 122, 0, 131,
 160, 103, 221, 12, 246, 65, 92, 94, 247, 121, 206, 78, 110, 123, 98, 156, 24, 6
2, 98, 160, 114, 236, 96, 204, 10, 193, 96, 103, 199, 227, 43, 0, 3, 6, 3, 253,
29, 64, 172, 58, 159, 129, 138, 64, 10, 28, 74, 234, 139, 154, 123, 222, 244, 10
0, 204, 106, 184, 144, 187, 55, 218, 44, 112, 88, 255, 250, 23, 108, 28, 71, 105
, 137, 150, 4, 123, 17, 61, 32, 162, 155, 229, 147, 31, 64, 124, 58, 109, 96, 23
8, 19, 27, 73, 205, 86, 208, 69, 252, 106, 132, 109, 66, 1, 112, 71, 239, 164, 5
3, 109, 86, 11, 45, 91, 110, 64, 143, 126, 187, 199, 78, 178, 72, 229, 131, 212,
 15, 251, 54, 141, 227, 217, 134, 60, 109, 115, 209, 197, 48, 230, 82, 156, 137,
 54, 23, 5, 13, 171, 154, 72, 41, 95, 124, 142, 249, 81, 182, 19, 50, 183, 174,
27, 210, 91, 137, 139, 136, 73, 4, 24, 17, 2, 0, 9, 5, 2, 78, 92, 197, 105, 2, 2
7, 12, 0, 10, 9, 16, 246, 222, 246, 220, 125, 179, 128, 38, 160, 111, 0, 160, 23
2, 114, 152, 33, 43, 238, 159, 130, 96, 226, 207, 163, 67, 157, 107, 39, 140, 16
1, 206, 46, 0, 159, 86, 227, 189, 247, 26, 20, 21, 223, 155, 33, 96, 108, 250, 2
14, 208, 190, 74, 17, 43, 197]}, _MaybeParseError(1194, [('message', 'end of inp
ut')]))
>>> pgp.checksum
[16177770, 16177770]
>>> pgp.block
({'q': [160, 1419741510835642352048304995189763742904405493959L], 'p': [1024, 11
27816910289527974061075044173534361876859840537664278626449706640702688418029301
77627320915504464134384919932726006749358191240819384430019080661316770630666273
73305307310915789743932633603785654698308793554266637396727988530315573294163969
5505278895830548483021403317423418546610032441185543867874125797847L], 'length':
 418, 'tag': 6, 'g': [1021, 1964849467881428710466800502308496822275241438855066
35238515330556083042068220950919284412399840170713771945402998891574208390341039
32135373724979642618657215923305914952852917206440714185965098155864193815365920
66817031665349355835965700904192728740918402151294857530330803724346461299359845
6219939378070626L], 'version': 4, 'algorithm': 17, 'time': 1314702697, 'y': [102
4, 10219928411694978026422720720306614004007482190710640189044175475269784538174
10827189912669049493898343302013241201652486175229135262490562087262759959706358
79842784809961205983751216640065755418899948380797160593911267435665817196921472
667773602697266619313590993808832086874091786796761538144131865973860894L]}, _Ma
ybeParseError(293, [('expected', None, 0)]))
>>>

Compass Belt

So, I made my own compass belt, like many others. I was inspired by the Wired article published a couple of years ago.



The core parts are a HMC6352 compass module connected to a 3.3V Arduino Pro Mini. A ULN2803 with eight Darlington drivers supply power to the vibrators positioned at even intervals around the belt. Everything is powered by a 9V battery.



I found some sample code to read the bearing from the compass, and then added the vibrator control myself.

#include <Wire.h> 

int compassAddress = 0x42 >> 1; // From datasheet compass address is 0x42
// shift the address 1 bit right, the Wire library only needs the 7
// most significant bits for the address
int reading = 0;

int bearing = 0;
int offset = 10;

void setup() 
{ 
  Wire.begin();       // join i2c bus (address optional for master) 
  Serial.begin(9600); // start serial communication at 9600bps 
  pinMode(13, OUTPUT);
  digitalWrite(13, HIGH);
  
  pinMode(2, OUTPUT);
  pinMode(3, OUTPUT);
  pinMode(4, OUTPUT);
  pinMode(5, OUTPUT);
  pinMode(6, OUTPUT);
  pinMode(7, OUTPUT);
  pinMode(8, OUTPUT);
  pinMode(9, OUTPUT);
} 

void loop() 
{ 
  // step 1: instruct sensor to read echoes 
  Wire.beginTransmission(compassAddress);  // transmit to device
  // the address specified in the datasheet is 66 (0x42) 
  // but i2c adressing uses the high 7 bits so it's 33 
  Wire.send('A');        // command sensor to measure angle  
  Wire.endTransmission(); // stop transmitting 

  // step 2: wait for readings to happen 
  delay(10); // datasheet suggests at least 6000 microseconds 

  // step 3: request reading from sensor 
  Wire.requestFrom(compassAddress, 2); // request 2 bytes from slave device #33 

  // step 4: receive reading from sensor 
  if (2 <= Wire.available()) // if two bytes were received 
  { 
    reading = Wire.receive();  // receive high byte (overwrites previous reading) 
    reading = reading << 8; // shift high byte to be high 8 bits 
    reading += Wire.receive(); // receive low byte as lower 8 bits 
    reading /= 10;
    Serial.println(reading); // print the reading
  } 

  // Activate vibrators
  offset = -offset;
  bearing = reading + 180 + offset;
  if (bearing > 359) bearing -= 360;
  if (bearing < 0) bearing += 360;
    
  digitalWrite(9, bearing >= 0 && bearing < 45 ? HIGH : LOW);
  digitalWrite(8, bearing >= 45 && bearing < 90 ? HIGH : LOW);
  digitalWrite(7, bearing >= 90 && bearing < 135 ? HIGH : LOW);
  digitalWrite(6, bearing >= 135 && bearing < 180 ? HIGH : LOW);
  digitalWrite(5, bearing >= 180 && bearing < 225 ? HIGH : LOW);
  digitalWrite(4, bearing >= 225 && bearing < 270 ? HIGH : LOW);
  digitalWrite(3, bearing >= 270 && bearing < 315 ? HIGH : LOW);
  digitalWrite(2, bearing >= 315 && bearing < 360 ? HIGH : LOW);

  delay(10);
    
  digitalWrite(9, LOW);
  digitalWrite(8, LOW);
  digitalWrite(7, LOW);
  digitalWrite(6, LOW);
  digitalWrite(5, LOW);
  digitalWrite(4, LOW);
  digitalWrite(3, LOW);
  digitalWrite(2, LOW);  
}

The vibrators are glued on velcro, to be easily adjustable for various waist sizes. The cables are nicely embedded in velcro as well.

One of my friends advised me not to go on the subway with this thing, unless I wanted to get arrested. May you live in interesting times.

Albumplayer - An MPD Web Client

I have written a simple web client for the Music Player Daemon (MPD). It is suitable to use on mobile devices, like my Android phone. It allows me to select artists from my music collection and play their music on my headless media frontend.

To do anything useful, Albumplayer requires an MPD instance. Install that first. Then, extract albumplayer.tgz where you want it located on your web server. Edit mpd.expect to connect to your MPD instance. (Change the IP address.) Albumplayer is written in bash, expect, grep, sed, awk, Python and PHP. Make sure you have them installed on your system before use.

I tried a couple of lightweight web clients before I wrote Albumplayer, but all seemed to require that I first defined playlists in MPD, which then could be played in the client. That is not how I want it to work. I want to select artists directly from the database and play their music in a straightforward manner.

Albumplayer is completely insecure. Don't run it on a public web server.

Update DB will not work in the general case, when it fails to finish before Expect terminates the connection. An opportunity for improvement.

Enjoy your music.

Software Architecture is a Fractal

The structure of the Internet is not fundamentally different from the structure of a peer, is not fundamentally different from the structure of an application, is not fundamentally different from the structure of a library, is not fundamentally different from the structure of an operating system, is not fundamentally different from the structure of a driver is not fundamentally different from the structure of a CPU.

Mapping Relational Data to JSON/HTTP

Information stored in a SQL database can efficiently be made available over a HTTP API with JSON as media type. This article describes a strategy to build a web representation for normalized relational data as it is found in a SQL database. When creating applications for the web, I don't have to abandon decades of best practices in relational data modeling, but can continue to build upon them.

An application layer between the database and the frontend consisting of JSON objects identified by URI:s serve two purposes. First, it formats the data nicely for a HTML+CSS+JS frontend, separating the data from the presentation. Second, it enables automated data processing as it becomes possible for various scripts and agents to access the data directly using standard web protocols and formats.

Each row in the database is represented by a JSON document, uniquely identified by an URI. Relationships between entities are represented by embedded JSON objects containing a link and additional data that is necessary for the frontend implementation to be efficient. Many to many relationships are represented both by a link and a node, where node is a reference to the object that is part of the relation managing the relationship between the entities.

This means that each entity and each entity instance is represented by an URI, and the operations GET, POST, PUT and DELETE can be used in a straightforward manner. Additionally, relations used to build many to many relationships between entities are also represented by URIs. This means that I don't need any special purpose POST commands on the participating entities to build and destroy many to many relationships. The normal pattern using POST to create, DELETE to remove and PUT to update relationships can be used.

I use the URI patterns http://<domain>/<tablename>/<primarykey>, and http://<domain>/<tablename>/<unique> where <unique> is a unique identifier for the row, not necessarily the primary key. I use the latter pattern when I want users to have the ability to create URI:s from the information they have about the object they are looking for. An alternative is to use search to find entities, but a publicly known URI pattern makes it simpler for humans to quickly reference objects. (Note that this simplification is for humans to use, not software clients, which will discover the URI:s when using the API.) The former pattern is used for relations which have no significance for humans, and are primarily used by software clients only.

Time for an example from a PDM system.



A product is described in zero or more documents. A product can also form a relationship with another product, where one is part of the other. The relationship between a product and its documents is one to many, and the relationship between products is many to many. Both relationships have a lifecycle property defining the state of the relationship. A product is identified by a class and a number. A document with the product identifier and a number. Each entity also has an autmatically assigned sequence number as primary key.

Product:
-----------------------
| id | class | number |
+----+-------+--------+
| 1  | AB    | 22     |
| 2  | AB    | 23     |

Where id is the primary key and <class, number> is unique.

Document:
---------------------------------------
| id | belongsto | number | lifecycle |
+----+-----------+--------+-----------+
| 1  | 1         | 445    | active    |
| 2  | 1         | 811    | obsolete  |

Where id is the primary key, belongsto a foreign product key and <belongsto, number> is unique.

Subproduct:
------------------------------
| parent | child | lifecycle |
+--------+-------+-----------+
| 1      | 2     | active    |

Which represents the many to many relationship between products where parent and child are foreign product keys and <parent, child> is unique.

JSON representations

http://mydomain.com/product/AB+22:
{
 "class": "AB",
 "number": 22,
 "name": "AB 22",
 "immutable": [
  "class",
  "number"
 ],
 "mutable": [
 ],
 "documents": [
  {"node": "http://mydomain.com/document/AB+22/445",
   "link": "http://mydomain.com/document/AB+22/445",
   "name": "AB 22 - 445",
   "lifecycle": "active"},
  {"node": "http://mydomain.com/document/AB+22/811",
   "link": "http://mydomain.com/document/AB+22/811",
   "name": "AB 22 - 811"},
   "lifecycle": "obsolete"}
 ],
 "children": [
  {"node": "http://mydomain.com/subproduct/1/2",
   "link": "http://mydomain.com/product/AB+23",
   "name": "AB 23",
   "lifecycle": "active"}
 ],
 "parents": [
 ]
}

http://mydomain.com/product/AB+23:
{
 "class": "AB",
 "number": 23,
 "name": "AB 23",
 "immutable": [
  "class",
  "number"
 ],
 "mutable": [
 ],
 "documents": [
 ],
 "children": [
 ],
 "parents": [
  {"node": "http://mydomain.com/subproduct/1/2",
   "link": "http://mydomain.com/product/AB+22",
   "name": "AB 22",
   "lifecycle": "active"}
 ]
}

http://mydomain.com/document/AB+22/445:
{
 "belongsto": {
  "node": "http://mydomain.com/document/AB+22/445",
  "link": "http://mydomain.com/product/AB+22",
  "name": "AB 22"
 },
 "number": 445,
 "name": "AB 22 - 445",
 "lifecycle: "active",
 "immutable": [
  "belongsto",
  "number"
 ],
 "mutable": [
  "lifecycle"
 ]
}

http://mydomain.com/document/AB+22/811:
{
 "belongsto": {
  "node": "http://mydomain.com/document/AB+22/811",
  "link": "http://mydomain.com/product/AB+22",
  "name": "AB 22"
 },
 "number": 811,
 "name": "AB 22 - 811",
 "lifecycle: "obsolete",
 "immutable": [
  "belongsto",
  "number"
 ],
 "mutable": [
  "lifecycle"
 ]
}

http://mydomain.com/subproduct/1/2:
{
 "parent": {
  "node": "http://mydomain.com/subproduct/1/2",
  "link": "http://mydomain.com/product/AB+22",
  "name": "AB 22"
 },
 "child": {
  "node": "http://mydomain.com/subproduct/1/2",
  "link": "http://mydomain.com/product/AB+23",
  "name": "AB 23"
 },
 "lifecycle: "active",
 "immutable": [
  "parent",
  "child"
 ],
 "mutable": [
  "lifecycle"
 ]
}

name is a human readable identifier, It is present in relationships to avoid unnecessary server communication when relationships are displayed for humans to navigate. lifecycle is present in the relationships to make it possible for the frontend to act differently depending on the current state. This is also motivated by efficiency reasons.

Product properties parents and children link directly to the related products. Ihe node property links to the corresponding subproduct. link is used for normal navigation and node when the relationship itself shall be updated or deleted. node is also present in product-document relationships to make it easier to write polymorphic frontend code.

New objects are created by POSTing to http://mydomain.com/product, http://mydomain.com/document and http://mydomain.com/subproduct. Objects are removed with DELETE on the object URI. Objects are updated by PUTing a new representation to the object URI. Changes will be rejected if they compromise data integrity. Relationships can not be changed by PUTing a modified product. They can only be changed through document and subproduct objects.

mutable and immutable exists to make it easier to write polymorphic frontend code. mutable contains the fields that are overwritten in a PUT operation. Both together contains the POST parameters to use when creating a new instance.

Manufacturing Software

Some say software doesn't have to be manufactured like physical products, it can simply be copied. But this is not entirely true. The first copy is unique and must be manufactured just like physical things.

To manufacture the first copy, we need to know how to do it. This is specified in the most important product document, the manufacturing instruction. This document contains everything you need to know to manufacture the first copy. For very simple software products, this is the only artifact the software engineer must produce during the construction phase.

If the software product is a little bigger, it quickly becomes impractical to have everything in one document, from routines, tools and build scripts to source code. The manufacturing instruction is then reduced to only contain the manufacturing process mixed with references to the source code, the tools to install and the build scripts to run.

From the reasoning above, I come to the conclusion that software construction is the act of turning requirements into a manufacturing instruction, which is then handed to manufacturing for the production of the first copy. For small products it is the constructing engineer himself who is responsible for the manufacturing. For larger products the manufacturing is handled by the system integrator.

A GPS logger with speedometer

To be able to contribute to the visionary OpenStreetMap project, I built myself a simple GPS logger. I added a speedometer as well, because I like to keep track of my speed when running or biking.



I had a Globalsat EM-411 GPS module already, a leftover from one of my numerous unfinished projects. The EM-411 only does one thing, and that is to output NMEA0183 encoded messages using 4800bps RS-232 with TTL levels. No initialization or control messages necessary.

I used a 5V Arduino Pro Mini as microcontroller, because it is very easy to program with the simple IDE and Java-like language.

A 512Kbit EEPROM with I2C interface is used to store the GPS data. I could not get hold of a PDIP8 version, but I managed to solder a SO8 directly to a DIL socket for easier prototyping. There are a couple of useful references on how to control a I2C EEPROM from the Arduino. Make sure you connect it to ANALOG pins 4 & 5, which I did not do until after many hours of reading and debugging.

The speed is displayed using two seven segment displays. The decimal dots are used to show memory usage. Due to a minor thinking error, I used one with common anode and one with common kathode, but if you decide to use identical displays, this can easily be compensated for in the display() routine.

A push button trigger a memory dump to the serial port on the Arduino, which I connect to a PC using a TTL-232R cable from FTDI. The button can also be used to clear the memory, if pressed during power on.

With a three second log interval, the device can log for about an hour before the memory is full, and that is also the amount of time a 9V, 200mAh rechargeable battery will power it before going empty.

When a $GPGGA message is received from the EM-411, it is stored in the EEPROM unaltered. The memory dump to PC is therefore just a plain playback of NMEA0183 messages. Using this approach, any NMEA0183 compatible software on the PC can read the logs without any conversions. I use GPSBabel to convert between NMEA0183 and GPX, the format required by OpenStreetMap.

When a $GPRMC message is received from the EM-411, the speed in knots is extracted and converted to km/h before being displayed.

#include <Wire.h>
#include <EEPROM.h>

#define LOGDELAY 3000
#define BUFLEN 100
#define PHASEDELAY 10


byte buttonPin = 4;
byte commonPin[2] = {3, 2};
byte segmentPin[8] = {9, 8, 7, 6, 13, 12, 11, 10};

byte matrix[70] = {1, 1, 1, 0, 1, 1, 1,
                   1, 0, 0, 0, 0, 0, 1,
                   1, 1, 0, 1, 1, 1, 0,
                   1, 1, 0, 1, 0, 1, 1,
                   1, 0, 1, 1, 0, 0, 1,
                   0, 1, 1, 1, 0, 1, 1,
                   0, 1, 1, 1, 1, 1, 1,
                   1, 1, 0, 0, 0, 0, 1,
                   1, 1, 1, 1, 1, 1, 1,
                   1, 1, 1, 1, 0, 1, 1};

int deviceaddress = 0x50;

byte GGA[6] = {'$', 'G', 'P', 'G', 'G', 'A'};
byte RMC[6] = {'$', 'G', 'P', 'R', 'M', 'C'};

byte phase = 0;
long nextphase = 0;
int data = 0;
int state = 0;

unsigned int abase = 0;
unsigned int logend = 0;

byte packet[BUFLEN];
byte length = 0;
byte cache[BUFLEN];
byte cachelength = 0;
byte cachepos = 0;

long time = 0;
long nextlog = 0;

void setup()
{
  Serial.begin(4800);
  Wire.begin();
  
  pinMode(commonPin[0], OUTPUT);
  pinMode(commonPin[1], OUTPUT);
  for (byte i = 0; i < 8; i++)
  {
    pinMode(segmentPin[i], OUTPUT);
  }
  
  pinMode(buttonPin, INPUT);
  digitalWrite(buttonPin, HIGH); // Pull up
  
  // Read memory pointers
  abase = EEPROM.read(0);
  abase <<= 8;
  abase += EEPROM.read(1);
  if (abase < 2 || abase > 510)
  {
    abase = 2;
    EEPROM.write(0, abase >> 8);
    EEPROM.write(1, abase & 0xFF);
  }
  logend = EEPROM.read(abase);
  logend <<= 8;
  logend += EEPROM.read(abase+1);

  // Clear log if button pressed
  if (digitalRead(buttonPin) == 0)
  {
    abase++;
    if (abase > 510)
    {
      abase = 2;
    }
    EEPROM.write(0, abase >> 8);
    EEPROM.write(1, abase & 0xFF);
    logend = 0;
    EEPROM.write(abase, logend >> 8);
    EEPROM.write(abase+1, logend & 0xFF);   
   
    while (digitalRead(buttonPin) == 0);
  }  
}

void loop()
{
  time = millis();
  
  // Display driver
  if (time > nextphase) {
    display();
    nextphase = time + PHASEDELAY;
  }

  // Dump log to serial port
  if (digitalRead(buttonPin) == 0)
  {
    unsigned int address;
    for (address = 0; address < logend; address++)
    {
      Serial.print(i2c_eeprom_read_byte(deviceaddress, address), BYTE);
    }
    if (digitalRead(buttonPin) == 0)
    {
      for (; address < 65535; address++)
      {
        Serial.print(i2c_eeprom_read_byte(deviceaddress, address), BYTE);
      }
      while (digitalRead(buttonPin) == 0);
    }
  }

  // Serial receive
  if (Serial.available() > 0)
  {
    int serialByte = Serial.read();
    packet[length++] = serialByte;
    if (serialByte == 10 || length >= BUFLEN)
    {
      parsePacket();
      length = 0;
    }
  }
  
  // EEPROM write
  if (cachepos > 0)
  {
    i2c_eeprom_write_byte(deviceaddress, logend+cachepos, cache[cachepos]);
    cachepos++;

    if (cachepos == cachelength)
    {
      logend += cachelength;
      EEPROM.write(abase, logend >> 8);
      EEPROM.write(abase+1, logend & 0xFF);
      cachepos = 0;
      cachelength = 0;
    }
  } 
}

void parsePacket()
{
  if (isGGA() && time > nextlog && cachepos == 0 && length > 50)
  {
    nextlog = time + LOGDELAY;
    
    if (logend > 65535-length)
    {
      state = 3;
      return;
    }
    else if (logend > 32768)
    {
      state = 2;
    }
    else
    {
      state = 1;
    }

    byte pos;
    for (pos = 0; pos < length; pos++)
    {
      cache[pos] = packet[pos];
    }
    cachelength = length;
    i2c_eeprom_write_byte(deviceaddress, logend+cachepos, cache[cachepos]);
    cachepos++;
  }
  else if (isRMC())
  {
    byte pos = find(0, ',', 7);
    float speed = parseFloat(pos);
    data = speed * 1.852 + 0.5;
  }
}

int parseInt(byte pos)
{
  int result = 0;
  while (packet[pos] >= '0' && packet[pos] <= '9')
  {
    result = result*10 + packet[pos] - '0';
    pos++;
  }
  return result;
}

float parseFloat(byte pos)
{
  float result = parseInt(pos);
  pos = find(pos, '.', 1);
  float decimals = parseInt(pos);
  byte end = find(pos, ',', 1);
  pos++;
  while (pos < end)
  {
    decimals /= 10;
    pos++;
  }
  return result + decimals;
}

byte find(byte pos, char sign, byte ncomma)
{
  while (ncomma > 0)
  {
    if (packet[pos] == sign)
    {
      ncomma--;
    }
    pos++;
  }
  return pos;
}

boolean isGGA()
{
  for (byte i = 0; i < 6; i++)
  {
    if (GGA[i] != packet[i])
    {
      return false;
    }
  }
  return true;
}

boolean isRMC()
{
  for (byte i = 0; i < 6; i++)
  {
    if (RMC[i] != packet[i])
    {
      return false;
    }
  }
  return true;
}

void display()
{
  byte digit = 0;

  digitalWrite(commonPin[0], 0);
  digitalWrite(commonPin[1], 1);
    
  if (phase == 0)
  {
    if (state & 0x02)
    {
      digitalWrite(segmentPin[7], 1);      
    }
    else
    {
      digitalWrite(segmentPin[7], 0);            
    }

    digit = (data % 100) / 10;
  }
  else
  {
    if (state & 0x01)
    {
      digitalWrite(segmentPin[7], 0);      
    }
    else
    {
      digitalWrite(segmentPin[7], 1);            
    }

    digit = data % 10;
  }
  for (byte i = 0; i < 7; i++)
  {
    digitalWrite(segmentPin[i], (phase ^ matrix[digit * 7 + i]) & 1);
  }
  
  if (phase == 0)
  {
    digitalWrite(commonPin[1], 0);
    phase = 1;
  }
  else
  {
    digitalWrite(commonPin[0], 1);
    phase = 0;
  } 
}

void i2c_eeprom_write_byte( int deviceaddress, unsigned int eeaddress, byte data ) {
  Wire.beginTransmission(deviceaddress);
  Wire.send((int)(eeaddress >> 8)); // MSB
  Wire.send((int)(eeaddress & 0xFF)); // LSB
  Wire.send((int)data);
  Wire.endTransmission();
  delay(10);
}

// WARNING: address is a page address, 6-bit end will wrap around
// also, data can be maximum of about 30 bytes, because the Wire library has a buffer of 32 bytes
void i2c_eeprom_write_page( int deviceaddress, unsigned int eeaddresspage, byte* data, byte length ) {
  Wire.beginTransmission(deviceaddress);
  Wire.send((int)(eeaddresspage >> 8)); // MSB
  Wire.send((int)(eeaddresspage & 0xFF)); // LSB
  byte c;
  for ( c = 0; c < length; c++)
    Wire.send(data[c]);
  Wire.endTransmission();
  delay(10);
}

byte i2c_eeprom_read_byte( int deviceaddress, unsigned int eeaddress ) {
  byte rdata = 0x0F;
  Wire.beginTransmission(deviceaddress);
  Wire.send((int)(eeaddress >> 8)); // MSB
  Wire.send((int)(eeaddress & 0xFF)); // LSB
  Wire.endTransmission();
  Wire.requestFrom(deviceaddress,1);
  if (Wire.available()) rdata = Wire.receive();
  return rdata;
}

// maybe let's not read more than 30 or 32 bytes at a time!
void i2c_eeprom_read_buffer( int deviceaddress, unsigned int eeaddress, byte *buffer, int length ) {
  Wire.beginTransmission(deviceaddress);
  Wire.send((int)(eeaddress >> 8)); // MSB
  Wire.send((int)(eeaddress & 0xFF)); // LSB
  Wire.endTransmission();
  Wire.requestFrom(deviceaddress,length);
  int c = 0;
  for ( c = 0; c < length; c++ )
    if (Wire.available()) buffer[c] = Wire.receive();
}

From the code we can deduce that the two displays should be connected in parallel to digital I/O 6, 7, 8, 9, 10, 11, 12 and 13. Digital 2 and 3 are display select signals, connected to the anode and kathode on the displays. The push button is connected to digital 4, with the internal pull up resistor enabled. EM-411 TX (pin 3) is connected to Arduino RxD.