SlideShare a Scribd company logo

ZeroMQ Is The Answer

I
Ian Barber

The document provides code examples and explanations for using ZeroMQ, covering various socket types and messaging patterns such as request/reply, publish/subscribe, and pipeline processing. It includes setups for server and client, socket definitions, and methods for sending and receiving messages using ZeroMQ's context and sockets in PHP. Additionally, links to helpful resources and community contributions are provided.

Technology
1 of 58
Downloaded 1,529 times
1
2
Most read
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
Most read
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
Most read
55
56
57
58
e r 3 e r s w /2 2 5 arb a n i ew ian b h e al k /v @ t r n/ t e .i m il.c o m is rb nd .co a a oi ir m B /j p g n p:/ /ph er@ Ia tt :/ rb h ttp a h n.b ia
“0MQ is unbelievably cool – if you haven’t got a project that needs it, make one up” jon gifford - loggly
ZeroMQ Is The Answer
ZeroMQ Is The Answer
ZeroMQ Is The Answer
queue esb pipeline async pub/sub gateway
request/response $ctx = new ZMQContext(); rep.php $server = new ZMQSocket($ctx, ZMQ::SOCKET_REP); $server->bind("tcp://*:5454"); while(true) { $message = $server->recv(); $server->send($message . " World"); }
request/response $ctx = new ZMQContext(); req.php $req = new ZMQSocket($ctx, ZMQ::SOCKET_REQ); $req->connect("tcp://localhost:5454"); $req->send("Hello"); echo $req->recv();
ZeroMQ Is The Answer
import zmq rep.py context = zmq.Context() server = context.socket(zmq.REP) server.connect("tcp://localhost:5455") while True: message = server.recv() print "Sending", message, "Worldn" server.send(message + " World")
Image: http://flickr.com/photos/sebastian_bergmann/3318754086
wget http://download.zeromq.org/ zeromq-2.1.1.tar.gz tar xvzf zeromq-2.1.1.tar.gz cd zeromq-2.1.1/ ./configure make sudo make install pear channel-discover pear.zero.mq pecl install zero.mq/zmq-beta echo "extension=zmq.so" > /etc/php.d/zmq.ini http://github.com/zeromq/zeromq2
atomic string multipart messaging
Post Office Image: http://www.flickr.com/photos/10804218@N00/4315282973 Post Box Image: http://www.flickr.com/photos/kenjonbro/3027166169
ZeroMQ Is The Answer
queue
queue $ctx = new ZMQContext(); $front = $ctx->getSocket(ZMQ::SOCKET_XREP); $back = $ctx->getSocket(ZMQ::SOCKET_XREQ); $front->bind('tcp://*:5454'); $back->bind ('tcp://*:5455'); $poll = new ZMQPoll(); $poll->add($front, ZMQ::POLL_IN); $poll->add($back, ZMQ::POLL_IN); $read = $write = array(); $snd = ZMQ::MODE_SNDMORE; $rcv = ZMQ::SOCKOPT_RCVMORE; queu e.php
while(true) { $events = $poll->poll($read, $write); foreach($read as $socket) { if($socket === $front) { do { $msg = $front->recv(); $more = $front->getSockOpt($rcv); $back->send($msg, $more ? $snd:0); } while($more); } else if($socket === $back) { do { $msg = $back->recv(); $more = $back->getSockOpt($rcv); $front->send($msg, $more ? $snd:0); } while($more); }}}
0MQ1 0MQ2 Socket STDIN POLL events
$ctx = new ZMQContext(); $sock = $ctx->getSocket(ZMQ::SOCKET_PULL); $sock->bind("tcp://*:5555"); $fh = fopen("php://stdin", 'r'); $poll = new ZMQPoll(); $poll->add($sock, ZMQ::POLL_IN); $poll->add($fh, ZMQ::POLL_IN); while(true) { $events = $poll->poll($read, $write); if($read[0] === $sock) { echo "ZMQ: ", $read[0]->recv(); } else { echo "STDIN: ", fgets($read[0]); }} poll.php
ZeroMQ Is The Answer
stable / unstable Image: http://www.flickr.com/photos/pelican/235461339/
pipeline
pipeline define("NUM_WORKERS", 10); for($i = 0; $i < NUM_WORKERS; $i++) { if(pcntl_fork() == 0) { `php work.php`; exit; } } con troller.php $ctx = new ZMQContext(); $work = $ctx->getSocket(ZMQ::SOCKET_PUSH); $ctrl = $ctx->getSocket(ZMQ::SOCKET_PUSH); $work->setSockOpt(ZMQ::SOCKOPT_HWM, 10); $ctrl->setSockOpt(ZMQ::SOCKOPT_HWM, 1);
$work->bind("ipc:///tmp/work"); $ctrl->bind("ipc:///tmp/control"); sleep(1); $fh = fopen('data.txt', 'r'); while($data = fgets($fh)) { $work->send($data); } for($i = 0; $i < NUM_WORKERS+1; $i++) { $ctrl->send("END"); }
work.php $ctx = new ZMQContext(); $work = $ctx->getSocket(ZMQ::SOCKET_PULL); $work->setSockOpt(ZMQ::SOCKOPT_HWM, 1); $work->connect("ipc:///tmp/work"); $sink = $ctx->getSocket(ZMQ::SOCKET_PUSH); $sink->connect("ipc:///tmp/results"); $ctrl = $ctx->getSocket(ZMQ::SOCKET_PULL); $ctrl->setSockOpt(ZMQ::SOCKOPT_HWM, 1); $ctrl->connect("ipc:///tmp/control"); $poll = new ZMQPoll(); $poll->add($work, ZMQ::POLL_IN); $read = $write = array();
while(true) { $ev = $poll->poll($read, $write, 5000); if($ev) { $message = $work->recv(); $sink->send(strlen($message)); } else { try { if($ctrl->recv(ZMQ::MODE_NOBLOCK)) { exit(); } } catch(ZMQException $e) { // noop } } }
sink.php $ctx = new ZMQContext(); $res = $ctx->getSocket(ZMQ::SOCKET_PULL); $res->bind("ipc:///tmp/results"); $ctrl = $ctx->getSocket(ZMQ::SOCKET_PULL); $ctrl->setSockOpt(ZMQ::SOCKOPT_HWM, 1); $ctrl->connect("ipc:///tmp/control"); $poll = new ZMQPoll(); $poll->add($res, ZMQ::POLL_IN); $read = $write = array(); $total = 0;
while(true) { $ev = $poll->poll($read, $write, 10000); if($ev) { $total += $res->recv(); } else { try { if($ctrl->recv(ZMQ::MODE_NOBLOCK)) { echo $total, PHP_EOL; exit(); } } catch (ZMQException $e) { //noop } } }
$ php controller.php $ php sink.php Starting Worker 0 39694 Starting Worker 1 Starting Worker 2 Starting Worker 3 Starting Worker 4 Starting Worker 5 Starting Worker 6 Starting Worker 7 Starting Worker 8 Starting Worker 9
filter chain
im age up loaded for proc essing
re-encoding scale out
resizing and quan tizing
fan-in for validation
watermark
pub/sub Image: http://www.flickr.com/photos/nikonvscanon/4519133003/
pub/sub
pub send “h “hi” i” i” “h sub sub sub i” “h ted ann ned
pub/sub $ctx = new ZMQContext(); $pub = $ctx->getSocket(ZMQ::SOCKET_PUB); $pub->bind('tcp://*:5566'); $pull = $ctx->getSocket(ZMQ::SOCKET_PULL); $pull->bind('tcp://*:5567'); while(true) { $message = $pull->recv(); $pub->send($message); } server.php
$name = htmlspecialchars($_POST['name']); $msg =htmlspecialchars($_POST['message']); $ctx = new ZMQContext(); $send = $ctx->getSocket(ZMQ::SOCKET_PUSH); $send->connect('tcp://localhost:5567'); if($msg == 'm:joined') { $send->send( "<em>" . $name . " has joined</em>"); } else { $send->send($name . ': ' . $msg); } send.php
$ctx = new ZMQContext(); $sub = $ctx->getSocket(ZMQ::SOCKET_SUB); $sub->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE,''); $sub->connect('tcp://localhost:5566'); $poll = new ZMQPoll(); $poll->add($sub, ZMQ::POLL_IN); $read = $wri = array(); while(true) { $ev = $poll->poll($read, $wri, 5000000); if($ev > 0) { echo "<script type='text/javascript'> parent.updateChat('"; echo $sub->recv() ."');</script>"; } ob_flush(); flush(); } ch at.php
ZeroMQ Is The Answer
sub sub user data pub pub pub web web web
$ctx = new ZMQContext(); $socket = $ctx->getSocket(ZMQ::SOCKET_PUB); $socket->connect("ipc:///tmp/usercache"); $socket->connect("ipc:///tmp/datacache"); $type = array('users', 'data'); while(true) { $socket->send($type[array_rand($type)], ZMQ::MODE_SNDMORE); $socket->send(rand(0, 12)); sleep(rand(0,3)); } cach e.php
$ctx = new ZMQContext(); $socket = $ctx->getSocket(ZMQ::SOCKET_SUB); $socket->setSockOpt( ZMQ::SOCKOPT_SUBSCRIBE, "users"); $socket->bind("ipc:///tmp/usercache"); while(true) { $cache = $socket->recv(); $request = $socket->recv(); echo "Clearing $cache $requestn"; } userlistener.php
types of transport inproc ipc tcp pgm
ZeroMQ Is The Answer
distro sub web client sub web sub event sub sub web pub distro sub web client sub db
$ctx = new ZMQContext(); $out = $ctx->getSocket(ZMQ::SOCKET_PUB); $out->setSockOpt(ZMQ::SOCKOPT_RATE, 10000); $out->connect("epgm://;239.192.0.1:7601"); $in = $ctx->getSocket(ZMQ::SOCKET_PULL); $in->bind("tcp://*:6767"); $device = new ZMQDevice( ZMQ::DEVICE_FORWARDER, $in, $out); eventhub.php
$ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_SUB); $in->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE, ''); $in->setSockOpt(ZMQ::SOCKOPT_RATE, 10000); $in->connect("epgm://;239.192.0.1:7601"); $out = $ctx->getSocket(ZMQ::SOCKET_PUB); $out->bind("ipc:///tmp/events"); $device = new ZMQDevice( ZMQ::DEVICE_FORWARDER, $in, $out); distro.php
$ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_SUB); for($i = 0; $i<100; $i++) { $in->setSockOpt( ZMQ::SOCKOPT_SUBSCRIBE, rand(100000, 999999)); } $in->connect("ipc:///tmp/events"); $i = 0; while($i++ < 1000) { $who = $in->recv(); $msg = $in->recv(); printf("%s %s %s", $who, $msg, PHP_EOL); } cli ent.php
push handler mongrel pub client client mongrel 2 http://mongrel2.org/
$ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_PULL); $in->connect('tcp://localhost:9997'); $out = $ctx->getSocket(ZMQ::SOCKET_PUB); $out->connect('tcp://localhost:9996'); $http = "HTTP/1.1 200 OKrnContent-Length: %srnrn%s"; while(true) { $msg = $in->recv(); list($uuid, $id, $path, $rest) = explode(" ", $msg, 4); $res = $uuid." ".strlen($id).':'.$id.", "; $res .= sprintf($http, 6, "Hello!"); $out->send($res); } handler.php
simple_handler = Handler( send_spec='tcp://*:9997', send_ident='ab206881-6f49-4276-9db1-1676bfae18b0', recv_spec='tcp://*:9996', recv_ident='' ) main = Server( uuid="9e71cabf-6afb-4ee1-b550-7972245f7e0a", access_log="/logs/access.log", error_log="/logs/error.log", chroot="./", default_host="general.local", name="example", pid_file="/run/mongre2.pid", port=6767, hosts = [ Host(name="general.local", routes={'/test':simple_handler}) ] ) settings = {"zeromq.threads": 1} servers = [main]
namespace m2php; $id ="82209006-86FF-4982-B5EA-D1E29E55D481"; $con = new m2phpConnection($id, "tcp://127.0.0.1:9997", "tcp://127.0.0.1:9996"); $ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_SUB); $in->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE,''); $sub->connect('tcp://localhost:5566'); $poll = new ZMQPoll(); $poll->add($sub, ZMQ::POLL_IN); $poll->add($con->reqs, ZMQ::POLL_IN); $read = $write = $ids = array(); $snd = ''; $h = "HTTP/1.1 200 OKrnContent-Type: text/ htmlrnTransfer-Encoding: chunkedrnrn"; https://github.com/winks/m2php
while (true) { $ev = $poll->poll($read, $write); foreach($read as $r) { if($r === $in) { $m = "<script type='text/javascript'> parent.updateChat('".$in->recv()."'); </script>rn"; $con->send($snd, implode(' ',$ids), sprintf("%xrn%s",strlen($m),$m)); } else { $req = $con->recv(); $snd = $req->sender; if($req->is_disconnect()) { unset($ids[$req->conn_id]); } else { $ids[$req->conn_id] = $req->conn_id; $con->send($snd, $req->conn_id,$h); } } } }
Helpful Links http://zero.mq http://zguide.zero.mq Ian Barber nbarber/ZeroMQ-Talk http://github.com/ia http://phpir.com ian.barber@gmail.com @ianbarber http://joind.in/talk/view/2523 thanks!

More Related Content

ODP
Overview of ZeroMQ
pieterh
 
KEY
Europycon2011: Implementing distributed application using ZeroMQ
fcrippa
 
KEY
ZeroMQ
Stoyan Zhekov
 
PDF
JSON Web Tokens
Ivan Rosolen
 
PDF
Nestjs MasterClass Slides
Nir Kaufman
 
PDF
Microservice With Spring Boot and Spring Cloud
Eberhard Wolff
 
PPTX
Deep Dive into Building a Secure & Multi-tenant SaaS Solution with NATS
NATS
 
PPTX
How to Load Custom Field to POS in Odoo 17 - Odoo 17 Slides
Celine George
 
Overview of ZeroMQ
pieterh
 
Europycon2011: Implementing distributed application using ZeroMQ
fcrippa
 
ZeroMQ
Stoyan Zhekov
 
JSON Web Tokens
Ivan Rosolen
 
Nestjs MasterClass Slides
Nir Kaufman
 
Microservice With Spring Boot and Spring Cloud
Eberhard Wolff
 
Deep Dive into Building a Secure & Multi-tenant SaaS Solution with NATS
NATS
 
How to Load Custom Field to POS in Odoo 17 - Odoo 17 Slides
Celine George
 

What's hot (20)

PDF
OIDC4VP for AB/C WG
Torsten Lodderstedt
 
PDF
Introduction to AMQP Messaging with RabbitMQ
Dmitriy Samovskiy
 
PDF
Using JSON Web Tokens for REST Authentication
Mediacurrent
 
PDF
Custom Applications with Spark's RDD: Spark Summit East talk by Tejas Patil
Spark Summit
 
PPTX
Solid design principles
Mahmoud Asadi
 
PDF
HTTP Security Headers
Ismael Goncalves
 
PPTX
Express JS
Alok Guha
 
PPTX
Vue Vuex 101
LocNguyen362
 
PDF
Introduction to Vagrant
Marcelo Pinheiro
 
PPTX
JMS Providers Overview
Vadym Lotar
 
PDF
K8s cluster autoscaler
k8s study
 
PPTX
Understanding JWT Exploitation
AkshaeyBhosale
 
PDF
Integrating Fiware Orion, Keyrock and Wilma
Dalton Valadares
 
PDF
SAML VS OAuth 2.0 VS OpenID Connect
Ubisecure
 
ODP
OAuth2 - Introduction
Knoldus Inc.
 
PDF
Express node js
Yashprit Singh
 
PDF
Network-Connected Development with ZeroMQ
ICS
 
PDF
Http security response headers
mohammadhosseinrouha
 
PDF
Jwt Security
Seid Yassin
 
PPT
Solid principles
Dmitry Kandalov
 
OIDC4VP for AB/C WG
Torsten Lodderstedt
 
Introduction to AMQP Messaging with RabbitMQ
Dmitriy Samovskiy
 
Using JSON Web Tokens for REST Authentication
Mediacurrent
 
Custom Applications with Spark's RDD: Spark Summit East talk by Tejas Patil
Spark Summit
 
Solid design principles
Mahmoud Asadi
 
HTTP Security Headers
Ismael Goncalves
 
Express JS
Alok Guha
 
Vue Vuex 101
LocNguyen362
 
Introduction to Vagrant
Marcelo Pinheiro
 
JMS Providers Overview
Vadym Lotar
 
K8s cluster autoscaler
k8s study
 
Understanding JWT Exploitation
AkshaeyBhosale
 
Integrating Fiware Orion, Keyrock and Wilma
Dalton Valadares
 
SAML VS OAuth 2.0 VS OpenID Connect
Ubisecure
 
OAuth2 - Introduction
Knoldus Inc.
 
Express node js
Yashprit Singh
 
Network-Connected Development with ZeroMQ
ICS
 
Http security response headers
mohammadhosseinrouha
 
Jwt Security
Seid Yassin
 
Solid principles
Dmitry Kandalov
 
Ad

Similar to ZeroMQ Is The Answer (20)

PDF
ZeroMQ Is The Answer: DPC 11 Version
Ian Barber
 
PDF
ZeroMQ Is The Answer: PHP Tek 11 Version
Ian Barber
 
KEY
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Masahiro Nagano
 
PDF
React PHP: the NodeJS challenger
vanphp
 
PDF
ZeroMQ: Messaging Made Simple
Ian Barber
 
KEY
Designing Opeation Oriented Web Applications / YAPC::Asia Tokyo 2011
Masahiro Nagano
 
PDF
Redis & ZeroMQ: How to scale your application
rjsmelo
 
PDF
20 modules i haven't yet talked about
Tatsuhiko Miyagawa
 
PDF
Symfony components in the wild, PHPNW12
Jakub Zalas
 
PPTX
PHP in 2018 - Q4 - AFUP Limoges
✅ William Pinaud
 
PDF
Micropage in microtime using microframework
Radek Benkel
 
ODP
The promise of asynchronous php
Wim Godden
 
PDF
Forget about index.php and build you applications around HTTP!
Kacper Gunia
 
PDF
Using ngx_lua in UPYUN
Cong Zhang
 
KEY
Perl Web Client
Flavio Poletti
 
PDF
Forget about Index.php and build you applications around HTTP - PHPers Cracow
Kacper Gunia
 
PDF
Javascript Continues Integration in Jenkins with AngularJS
Ladislav Prskavec
 
PDF
ReactPHP
Philip Norton
 
KEY
Operation Oriented Web Applications / Yokohama pm7
Masahiro Nagano
 
KEY
Mojo as a_client
Marcus Ramberg
 
ZeroMQ Is The Answer: DPC 11 Version
Ian Barber
 
ZeroMQ Is The Answer: PHP Tek 11 Version
Ian Barber
 
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Masahiro Nagano
 
React PHP: the NodeJS challenger
vanphp
 
ZeroMQ: Messaging Made Simple
Ian Barber
 
Designing Opeation Oriented Web Applications / YAPC::Asia Tokyo 2011
Masahiro Nagano
 
Redis & ZeroMQ: How to scale your application
rjsmelo
 
20 modules i haven't yet talked about
Tatsuhiko Miyagawa
 
Symfony components in the wild, PHPNW12
Jakub Zalas
 
PHP in 2018 - Q4 - AFUP Limoges
✅ William Pinaud
 
Micropage in microtime using microframework
Radek Benkel
 
The promise of asynchronous php
Wim Godden
 
Forget about index.php and build you applications around HTTP!
Kacper Gunia
 
Using ngx_lua in UPYUN
Cong Zhang
 
Perl Web Client
Flavio Poletti
 
Forget about Index.php and build you applications around HTTP - PHPers Cracow
Kacper Gunia
 
Javascript Continues Integration in Jenkins with AngularJS
Ladislav Prskavec
 
ReactPHP
Philip Norton
 
Operation Oriented Web Applications / Yokohama pm7
Masahiro Nagano
 
Mojo as a_client
Marcus Ramberg
 
Ad

More from Ian Barber (10)

PDF
How to stand on the shoulders of giants
Ian Barber
 
PDF
Teaching Your Machine To Find Fraudsters
Ian Barber
 
PDF
Debugging: Rules And Tools - PHPTek 11 Version
Ian Barber
 
PDF
Deployment Tactics
Ian Barber
 
PDF
In Search Of: Integrating Site Search (PHP Barcelona)
Ian Barber
 
PDF
Debugging: Rules & Tools
Ian Barber
 
PDF
In Search Of... (Dutch PHP Conference 2010)
Ian Barber
 
PDF
In Search Of... integrating site search
Ian Barber
 
KEY
Document Classification In PHP - Slight Return
Ian Barber
 
KEY
Document Classification In PHP
Ian Barber
 
How to stand on the shoulders of giants
Ian Barber
 
Teaching Your Machine To Find Fraudsters
Ian Barber
 
Debugging: Rules And Tools - PHPTek 11 Version
Ian Barber
 
Deployment Tactics
Ian Barber
 
In Search Of: Integrating Site Search (PHP Barcelona)
Ian Barber
 
Debugging: Rules & Tools
Ian Barber
 
In Search Of... (Dutch PHP Conference 2010)
Ian Barber
 
In Search Of... integrating site search
Ian Barber
 
Document Classification In PHP - Slight Return
Ian Barber
 
Document Classification In PHP
Ian Barber
 

Recently uploaded (20)

PDF
Unlocking AI with Model Context Protocol (MCP)
Brian McKeiver
 
PDF
Getting Started with Data Integration: FME Form 101
Safe Software
 
PDF
1 - Historical Antecedents, Social Consideration.pdf
tuazon2030627
 
PDF
DASA ADMISSION 2024_FirstRound_FirstRank_LastRank.pdf
rameswartudu05
 
PPTX
Tartificialntelligence_presentation.pptx
ry7r52mzb4
 
PDF
Web App vs Mobile App What Should You Build First.pdf
KodekX
 
PDF
Transform Your ITIL® 4 & ITSM Strategy with AI in 2025.pdf
PDCA Consulting
 
PDF
MIND Revenue Release Quarter 2 2025 Press Release
MIND CTI
 
PDF
A comparative analysis of optical character recognition models for extracting...
IAESIJAI
 
PDF
Mushroom cultivation and it's methods.pdf
mailmeonrishi
 
PDF
WOOl fibre morphology and structure.pdf for textiles
Rajendrakumar868651
 
PDF
Hybrid model detection and classification of lung cancer
IAESIJAI
 
PPTX
SOPHOS-XG Firewall Administrator PPT.pptx
AgnesMunisi
 
PDF
Assigned Numbers - 2025 - Bluetooth® Document
Bloombase
 
PPTX
Group 1 Presentation -Planning and Decision Making .pptx
MatthewLewis227954
 
PPTX
Chapter 5: Probability Theory and Statistics
RandyFin
 
PDF
ENT215_Completing-a-large-scale-migration-and-modernization-with-AWS.pdf
ssuser28425f
 
PDF
DP Operators-handbook-extract for the Mautical Institute
LeonelMejiaLarios
 
PPTX
1. Introduction to Computer Programming.pptx
bonakiduza1024
 
PPTX
Programs and apps: productivity, graphics, security and other tools
4mqw9zch22
 
Unlocking AI with Model Context Protocol (MCP)
Brian McKeiver
 
Getting Started with Data Integration: FME Form 101
Safe Software
 
1 - Historical Antecedents, Social Consideration.pdf
tuazon2030627
 
DASA ADMISSION 2024_FirstRound_FirstRank_LastRank.pdf
rameswartudu05
 
Tartificialntelligence_presentation.pptx
ry7r52mzb4
 
Web App vs Mobile App What Should You Build First.pdf
KodekX
 
Transform Your ITIL® 4 & ITSM Strategy with AI in 2025.pdf
PDCA Consulting
 
MIND Revenue Release Quarter 2 2025 Press Release
MIND CTI
 
A comparative analysis of optical character recognition models for extracting...
IAESIJAI
 
Mushroom cultivation and it's methods.pdf
mailmeonrishi
 
WOOl fibre morphology and structure.pdf for textiles
Rajendrakumar868651
 
Hybrid model detection and classification of lung cancer
IAESIJAI
 
SOPHOS-XG Firewall Administrator PPT.pptx
AgnesMunisi
 
Assigned Numbers - 2025 - Bluetooth® Document
Bloombase
 
Group 1 Presentation -Planning and Decision Making .pptx
MatthewLewis227954
 
Chapter 5: Probability Theory and Statistics
RandyFin
 
ENT215_Completing-a-large-scale-migration-and-modernization-with-AWS.pdf
ssuser28425f
 
DP Operators-handbook-extract for the Mautical Institute
LeonelMejiaLarios
 
1. Introduction to Computer Programming.pptx
bonakiduza1024
 
Programs and apps: productivity, graphics, security and other tools
4mqw9zch22
 

ZeroMQ Is The Answer

  • 1. e r 3 e r s w /2 2 5 arb a n i ew ian b h e al k /v @ t r n/ t e .i m il.c o m is rb nd .co a a oi ir m B /j p g n p:/ /ph er@ Ia tt :/ rb h ttp a h n.b ia
  • 2. “0MQ is unbelievably cool – if you haven’t got a project that needs it, make one up” jon gifford - loggly
  • 6. queue esb pipeline async pub/sub gateway
  • 7. request/response $ctx = new ZMQContext(); rep.php $server = new ZMQSocket($ctx, ZMQ::SOCKET_REP); $server->bind("tcp://*:5454"); while(true) { $message = $server->recv(); $server->send($message . " World"); }
  • 8. request/response $ctx = new ZMQContext(); req.php $req = new ZMQSocket($ctx, ZMQ::SOCKET_REQ); $req->connect("tcp://localhost:5454"); $req->send("Hello"); echo $req->recv();
  • 10. import zmq rep.py context = zmq.Context() server = context.socket(zmq.REP) server.connect("tcp://localhost:5455") while True: message = server.recv() print "Sending", message, "Worldn" server.send(message + " World")
  • 12. wget http://download.zeromq.org/ zeromq-2.1.1.tar.gz tar xvzf zeromq-2.1.1.tar.gz cd zeromq-2.1.1/ ./configure make sudo make install pear channel-discover pear.zero.mq pecl install zero.mq/zmq-beta echo "extension=zmq.so" > /etc/php.d/zmq.ini http://github.com/zeromq/zeromq2
  • 13. atomic string multipart messaging
  • 14. Post Office Image: http://www.flickr.com/photos/10804218@N00/4315282973 Post Box Image: http://www.flickr.com/photos/kenjonbro/3027166169
  • 16. queue
  • 17. queue $ctx = new ZMQContext(); $front = $ctx->getSocket(ZMQ::SOCKET_XREP); $back = $ctx->getSocket(ZMQ::SOCKET_XREQ); $front->bind('tcp://*:5454'); $back->bind ('tcp://*:5455'); $poll = new ZMQPoll(); $poll->add($front, ZMQ::POLL_IN); $poll->add($back, ZMQ::POLL_IN); $read = $write = array(); $snd = ZMQ::MODE_SNDMORE; $rcv = ZMQ::SOCKOPT_RCVMORE; queu e.php
  • 18. while(true) { $events = $poll->poll($read, $write); foreach($read as $socket) { if($socket === $front) { do { $msg = $front->recv(); $more = $front->getSockOpt($rcv); $back->send($msg, $more ? $snd:0); } while($more); } else if($socket === $back) { do { $msg = $back->recv(); $more = $back->getSockOpt($rcv); $front->send($msg, $more ? $snd:0); } while($more); }}}
  • 19. 0MQ1 0MQ2 Socket STDIN POLL events
  • 20. $ctx = new ZMQContext(); $sock = $ctx->getSocket(ZMQ::SOCKET_PULL); $sock->bind("tcp://*:5555"); $fh = fopen("php://stdin", 'r'); $poll = new ZMQPoll(); $poll->add($sock, ZMQ::POLL_IN); $poll->add($fh, ZMQ::POLL_IN); while(true) { $events = $poll->poll($read, $write); if($read[0] === $sock) { echo "ZMQ: ", $read[0]->recv(); } else { echo "STDIN: ", fgets($read[0]); }} poll.php
  • 22. stable / unstable Image: http://www.flickr.com/photos/pelican/235461339/
  • 24. pipeline define("NUM_WORKERS", 10); for($i = 0; $i < NUM_WORKERS; $i++) { if(pcntl_fork() == 0) { `php work.php`; exit; } } con troller.php $ctx = new ZMQContext(); $work = $ctx->getSocket(ZMQ::SOCKET_PUSH); $ctrl = $ctx->getSocket(ZMQ::SOCKET_PUSH); $work->setSockOpt(ZMQ::SOCKOPT_HWM, 10); $ctrl->setSockOpt(ZMQ::SOCKOPT_HWM, 1);
  • 25. $work->bind("ipc:///tmp/work"); $ctrl->bind("ipc:///tmp/control"); sleep(1); $fh = fopen('data.txt', 'r'); while($data = fgets($fh)) { $work->send($data); } for($i = 0; $i < NUM_WORKERS+1; $i++) { $ctrl->send("END"); }
  • 26. work.php $ctx = new ZMQContext(); $work = $ctx->getSocket(ZMQ::SOCKET_PULL); $work->setSockOpt(ZMQ::SOCKOPT_HWM, 1); $work->connect("ipc:///tmp/work"); $sink = $ctx->getSocket(ZMQ::SOCKET_PUSH); $sink->connect("ipc:///tmp/results"); $ctrl = $ctx->getSocket(ZMQ::SOCKET_PULL); $ctrl->setSockOpt(ZMQ::SOCKOPT_HWM, 1); $ctrl->connect("ipc:///tmp/control"); $poll = new ZMQPoll(); $poll->add($work, ZMQ::POLL_IN); $read = $write = array();
  • 27. while(true) { $ev = $poll->poll($read, $write, 5000); if($ev) { $message = $work->recv(); $sink->send(strlen($message)); } else { try { if($ctrl->recv(ZMQ::MODE_NOBLOCK)) { exit(); } } catch(ZMQException $e) { // noop } } }
  • 28. sink.php $ctx = new ZMQContext(); $res = $ctx->getSocket(ZMQ::SOCKET_PULL); $res->bind("ipc:///tmp/results"); $ctrl = $ctx->getSocket(ZMQ::SOCKET_PULL); $ctrl->setSockOpt(ZMQ::SOCKOPT_HWM, 1); $ctrl->connect("ipc:///tmp/control"); $poll = new ZMQPoll(); $poll->add($res, ZMQ::POLL_IN); $read = $write = array(); $total = 0;
  • 29. while(true) { $ev = $poll->poll($read, $write, 10000); if($ev) { $total += $res->recv(); } else { try { if($ctrl->recv(ZMQ::MODE_NOBLOCK)) { echo $total, PHP_EOL; exit(); } } catch (ZMQException $e) { //noop } } }
  • 30. $ php controller.php $ php sink.php Starting Worker 0 39694 Starting Worker 1 Starting Worker 2 Starting Worker 3 Starting Worker 4 Starting Worker 5 Starting Worker 6 Starting Worker 7 Starting Worker 8 Starting Worker 9
  • 32. im age up loaded for proc essing
  • 34. resizing and quan tizing
  • 37. pub/sub Image: http://www.flickr.com/photos/nikonvscanon/4519133003/
  • 39. pub send “h “hi” i” i” “h sub sub sub i” “h ted ann ned
  • 40. pub/sub $ctx = new ZMQContext(); $pub = $ctx->getSocket(ZMQ::SOCKET_PUB); $pub->bind('tcp://*:5566'); $pull = $ctx->getSocket(ZMQ::SOCKET_PULL); $pull->bind('tcp://*:5567'); while(true) { $message = $pull->recv(); $pub->send($message); } server.php
  • 41. $name = htmlspecialchars($_POST['name']); $msg =htmlspecialchars($_POST['message']); $ctx = new ZMQContext(); $send = $ctx->getSocket(ZMQ::SOCKET_PUSH); $send->connect('tcp://localhost:5567'); if($msg == 'm:joined') { $send->send( "<em>" . $name . " has joined</em>"); } else { $send->send($name . ': ' . $msg); } send.php
  • 42. $ctx = new ZMQContext(); $sub = $ctx->getSocket(ZMQ::SOCKET_SUB); $sub->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE,''); $sub->connect('tcp://localhost:5566'); $poll = new ZMQPoll(); $poll->add($sub, ZMQ::POLL_IN); $read = $wri = array(); while(true) { $ev = $poll->poll($read, $wri, 5000000); if($ev > 0) { echo "<script type='text/javascript'> parent.updateChat('"; echo $sub->recv() ."');</script>"; } ob_flush(); flush(); } ch at.php
  • 44. sub sub user data pub pub pub web web web
  • 45. $ctx = new ZMQContext(); $socket = $ctx->getSocket(ZMQ::SOCKET_PUB); $socket->connect("ipc:///tmp/usercache"); $socket->connect("ipc:///tmp/datacache"); $type = array('users', 'data'); while(true) { $socket->send($type[array_rand($type)], ZMQ::MODE_SNDMORE); $socket->send(rand(0, 12)); sleep(rand(0,3)); } cach e.php
  • 46. $ctx = new ZMQContext(); $socket = $ctx->getSocket(ZMQ::SOCKET_SUB); $socket->setSockOpt( ZMQ::SOCKOPT_SUBSCRIBE, "users"); $socket->bind("ipc:///tmp/usercache"); while(true) { $cache = $socket->recv(); $request = $socket->recv(); echo "Clearing $cache $requestn"; } userlistener.php
  • 47. types of transport inproc ipc tcp pgm
  • 49. distro sub web client sub web sub event sub sub web pub distro sub web client sub db
  • 50. $ctx = new ZMQContext(); $out = $ctx->getSocket(ZMQ::SOCKET_PUB); $out->setSockOpt(ZMQ::SOCKOPT_RATE, 10000); $out->connect("epgm://;239.192.0.1:7601"); $in = $ctx->getSocket(ZMQ::SOCKET_PULL); $in->bind("tcp://*:6767"); $device = new ZMQDevice( ZMQ::DEVICE_FORWARDER, $in, $out); eventhub.php
  • 51. $ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_SUB); $in->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE, ''); $in->setSockOpt(ZMQ::SOCKOPT_RATE, 10000); $in->connect("epgm://;239.192.0.1:7601"); $out = $ctx->getSocket(ZMQ::SOCKET_PUB); $out->bind("ipc:///tmp/events"); $device = new ZMQDevice( ZMQ::DEVICE_FORWARDER, $in, $out); distro.php
  • 52. $ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_SUB); for($i = 0; $i<100; $i++) { $in->setSockOpt( ZMQ::SOCKOPT_SUBSCRIBE, rand(100000, 999999)); } $in->connect("ipc:///tmp/events"); $i = 0; while($i++ < 1000) { $who = $in->recv(); $msg = $in->recv(); printf("%s %s %s", $who, $msg, PHP_EOL); } cli ent.php
  • 53. push handler mongrel pub client client mongrel 2 http://mongrel2.org/
  • 54. $ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_PULL); $in->connect('tcp://localhost:9997'); $out = $ctx->getSocket(ZMQ::SOCKET_PUB); $out->connect('tcp://localhost:9996'); $http = "HTTP/1.1 200 OKrnContent-Length: %srnrn%s"; while(true) { $msg = $in->recv(); list($uuid, $id, $path, $rest) = explode(" ", $msg, 4); $res = $uuid." ".strlen($id).':'.$id.", "; $res .= sprintf($http, 6, "Hello!"); $out->send($res); } handler.php
  • 55. simple_handler = Handler( send_spec='tcp://*:9997', send_ident='ab206881-6f49-4276-9db1-1676bfae18b0', recv_spec='tcp://*:9996', recv_ident='' ) main = Server( uuid="9e71cabf-6afb-4ee1-b550-7972245f7e0a", access_log="/logs/access.log", error_log="/logs/error.log", chroot="./", default_host="general.local", name="example", pid_file="/run/mongre2.pid", port=6767, hosts = [ Host(name="general.local", routes={'/test':simple_handler}) ] ) settings = {"zeromq.threads": 1} servers = [main]
  • 56. namespace m2php; $id ="82209006-86FF-4982-B5EA-D1E29E55D481"; $con = new m2phpConnection($id, "tcp://127.0.0.1:9997", "tcp://127.0.0.1:9996"); $ctx = new ZMQContext(); $in = $ctx->getSocket(ZMQ::SOCKET_SUB); $in->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE,''); $sub->connect('tcp://localhost:5566'); $poll = new ZMQPoll(); $poll->add($sub, ZMQ::POLL_IN); $poll->add($con->reqs, ZMQ::POLL_IN); $read = $write = $ids = array(); $snd = ''; $h = "HTTP/1.1 200 OKrnContent-Type: text/ htmlrnTransfer-Encoding: chunkedrnrn"; https://github.com/winks/m2php
  • 57. while (true) { $ev = $poll->poll($read, $write); foreach($read as $r) { if($r === $in) { $m = "<script type='text/javascript'> parent.updateChat('".$in->recv()."'); </script>rn"; $con->send($snd, implode(' ',$ids), sprintf("%xrn%s",strlen($m),$m)); } else { $req = $con->recv(); $snd = $req->sender; if($req->is_disconnect()) { unset($ids[$req->conn_id]); } else { $ids[$req->conn_id] = $req->conn_id; $con->send($snd, $req->conn_id,$h); } } } }
  • 58. Helpful Links http://zero.mq http://zguide.zero.mq Ian Barber nbarber/ZeroMQ-Talk http://github.com/ia http://phpir.com [email protected] @ianbarber http://joind.in/talk/view/2523 thanks!