mfabrik-blog.html | searchcode

/tests/mfabrik-blog.html

https://github.com/aaitxcoder/mobilize.js
HTML | 1094 lines | 969 code | 118 blank | 7 comment | 0 complexity | 7d56177c5fbd7d2d3774e84e7458238e MD5 | raw file

<!DOCTYPE html>
<!-- Test mobilize.js against another blog with lots of content -->
<html dir="ltr" lang="en-US">
<head>
<meta charset="UTF-8" />
<title>mFabrik - mobile sites, apps, HTML5 and CMS software development</title>
<link rel="profile" href="http://gmpg.org/xfn/11" />
<link rel="stylesheet" type="text/css" media="all" href="http://blog.mfabrik.com/wp-content/themes/twentyten/style.css" />
<link rel="pingback" href="http://blog.mfabrik.com/xmlrpc.php" />
<link rel="alternate" type="application/rss+xml" title="mFabrik - mobile sites, apps, HTML5 and CMS software development &raquo; Feed" href="http://blog.mfabrik.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="mFabrik - mobile sites, apps, HTML5 and CMS software development &raquo; Comments Feed" href="http://blog.mfabrik.com/comments/feed/" />

<link rel='stylesheet' id='shashin_css-css'  href='http://blog.mfabrik.com/wp-content/plugins/shashin/display/shashin.css?ver=2.6.3' type='text/css' media='all' />

<link rel='stylesheet' id='highslide_css-css'  href='http://blog.mfabrik.com/wp-content/plugins/shashin/display/highslide.css?ver=4.1.4' type='text/css' media='all' />

<script type='text/javascript' src='http://blog.mfabrik.com/wp-includes/js/l10n.js?ver=20101110'></script>

<script type='text/javascript'>
/* <![CDATA[ */
var highslide_settings = {
	graphics_dir: "http://blog.mfabrik.com/wp-content/plugins/shashin/display/highslide/graphics/",
	outline_type: "rounded-white",
	dimming_opacity: "0.75",
	interval: "5000",
	repeat: "1",
	position: "top center",
	hide_controller: "0"
};
/* ]]> */
</script>
<script type='text/javascript' src='http://blog.mfabrik.com/wp-content/plugins/shashin/display/highslide_settings.js?ver=2.6.3'></script>
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="http://blog.mfabrik.com/xmlrpc.php?rsd" />
<link rel="wlwmanifest" type="application/wlwmanifest+xml" href="http://blog.mfabrik.com/wp-includes/wlwmanifest.xml" /> 
<link rel='index' title='mFabrik &#8211; mobile sites, apps, HTML5 and CMS software development' href='http://blog.mfabrik.com/' />
<meta name="generator" content="WordPress 3.1" />

<!-- All in One SEO Pack 1.6.13.1 by Michael Torbert of Semper Fi Web Design[78,176] -->
<meta name="keywords" content="all files,cyberduck,doc,download,drag and drop,export,import,mass upload,osx,pdf,photos,plone,webdav,answer,discussion,forum,irc,mailing list,python,question,smart questions,apache,error_log,php,php.ini,tail,xampp,chrome,commit hook,compress,console,firefox,git,github,ie,internet explorer,javascript,jquery,preprocessor,safari,webkit,app engine,data,mirror,remote api,sqlite,upload,always on,appengine,cloud,cloud computing,cron,deadlineexceedederror,dynamic,gaeutilities,google,http requiest,instance,monitoring,new process,response time,zabbix,ironpython,jython,microsoft,net,pypy,python tools for visual studio,adapter,interface,page template,portlet,render,zope,ajax,comments,disqus,lazy load,static,varnish,crawl,find,follow,full text,full text search,pypdf,scrape,scrapy,search,trademark,user policy,violation,web crawler" />
<link rel="canonical" href="http://blog.mfabrik.com/" />
<!-- /all in one seo pack -->
				
	<link type="text/css" rel="stylesheet" href="http://blog.mfabrik.com/wp-content/plugins/syntax/Styles/SyntaxHighlighter.css" />
</head>

<body class="home blog logged-in">
	
    <body>

        <script type="text/javascript">
	         // Don't start executing mobilize whilst loaading JS file, but wait
	         // for our manual (development commands)
	         window.mobilizeAutoload = false;
			 
			 function mobilizeCustomInit() {
			 	// Include mFabrik specific Javascript initialization layer
			 	mobilize.cdnOptions.javascriptBundles.push("http://localhost:8080/js/mobilize.mfabrik-blog.js");
			 }
        </script>
  
        <script class="mobilize-js-source" 
                type="text/javascript" 
                src="http://localhost:8080/js/mobilize.js"
                >
        </script>

        <script type="text/javascript" 
                src="http://localhost:8080/js/mobilize.wordpress.js"
                >
        </script>

        <script type="text/javascript">        

             // Setup mobilize.js to load our additional mobilize.mfabrik-blog.js layer
             function setupCustomBlog(){

                 mobilize.init({
                 }, {
                     // Don't do cloud error reporting
                     // (it would useful for production deployment only)
                     errorReportingURL: false,
                     
                     baseURL: "http://localhost:8080", // Test server

                     // Load JS files locally, unbundled
                     javascriptBundles : ["js/jquery.js", 
                                         "js/mobilize.onjq.js", 
                                         "js/jquery.mobile.js",
										 ],

                     // Load CSS files locally, unbundled
                     cssBundles: ["css/jquery.mobile.css", 
                                 "css/wordpress.css"],

                                          
                 });
				                  
                 // Since we are not in auto-run mode,
                 // we start doing the stuff after we have set-up
                 // our options for development correctly
                 mobilize.bootstrap();
                 
             }        
             setupCustomBlog();              
        </script>

<div id="wrapper" class="hfeed">
	<div id="header">
		<div id="masthead">
			<div id="branding" role="banner">
								<h1 id="site-title">
					<span>
						<a href="http://blog.mfabrik.com/" title="mFabrik &#8211; mobile sites, apps, HTML5 and CMS software development" rel="home">mFabrik &#8211; mobile sites, apps, HTML5 and CMS software development</a>
					</span>
				</h1>
				<div id="site-description">Freedom delivered.</div>

										<img src="http://blog.mfabrik.com/wp-content/themes/twentyten/images/headers/path.jpg" width="940" height="198" alt="" />
								</div><!-- #branding -->

			<div id="access" role="navigation">
			  				<div class="skip-link screen-reader-text"><a href="#content" title="Skip to content">Skip to content</a></div>
								<div class="menu"><ul><li class="current_page_item"><a href="http://blog.mfabrik.com/" title="Home">Home</a></li><li class="page_item page-item-2"><a href="http://blog.mfabrik.com/about/" title="About this blog">About this blog</a></li><li class="page_item page-item-336"><a href="http://blog.mfabrik.com/phonegap-on-maemo/" title="PhoneGap on Maemo">PhoneGap on Maemo</a></li></ul></div>
			</div><!-- #access -->
		</div><!-- #masthead -->
	</div><!-- #header -->

	<div id="main">

		<div id="container">
			<div id="content" role="main">

			
	<div id="nav-above" class="navigation">
		<div class="nav-previous"><a href="http://blog.mfabrik.com/page/2/" ><span class="meta-nav">&larr;</span> Older posts</a></div>
		<div class="nav-next"></div>
	</div><!-- #nav-above -->




			<div id="post-1133" class="post-1133 post type-post status-publish format-standard hentry category-plone category-technology tag-all-files tag-cyberduck tag-doc tag-download tag-drag-and-drop tag-export tag-import tag-mass-upload tag-osx tag-pdf tag-photos tag-plone tag-webdav">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/18/making-an-offline-file-system-copy-of-a-plone-site-using-webdav/" title="Permalink to Making an offline file system copy of a Plone site using WebDAV" rel="bookmark">Making an offline file system copy of a Plone site using WebDAV</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/18/making-an-offline-file-system-copy-of-a-plone-site-using-webdav/" title="12:06 pm" rel="bookmark"><span class="entry-date">March 18, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>You might want to create an offline copy of a Plone site because</p>
<ul>
<li>You are traveling and you want to have all files on the site (PDFs for reading)</li>
<li>You are taking a site down and making the final back-up</li>
<li>You just want to feel how cool Plone is</li>
</ul>
<p><a href="http://plone.org/documentation/kb/webdav">Plone supports WebDAV</a>.</p>
<p>Creating a file system viewable offline copy of a Plone site is a task of</p>
<ul>
<li><a href="http://plone.org/documentation/kb/webdav">Enabling WebDAV</a></li>
<li>Login to site via WebDAV. On OSX use <a href="http://cyberduck.ch/">Cyberduck</a>, Finder (the file browser of OSX itself) may have issues, though works. You might need Zope admin priviledges for certain operations.</li>
<li>Drag and drop Plone site to your hard disk</li>
</ul>
<p>WebDAV copy process works smoothly</p>
<ul>
<li>Folder and page structure is intact</li>
<li>Files are copied as is (think PDFs, Docs)</li>
<li>Images are copied as is</li>
<li>Pages (HTML) are converted to special files, which are still readable in plain-text editor</li>
</ul>
<p>Note that some special folders (acl_users, reference_catalog, etc.) might be exposed through WebDAV, but they are not really copyable. Just ignore these during the copy process.</p>
<p>You can also use WebDAV to mass upload files and images for your image bank instead of manually uploading them through web interface.
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/plone/" title="View all posts in plone" rel="category tag">plone</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/all-files/" rel="tag">all files</a>, <a href="http://blog.mfabrik.com/tag/cyberduck/" rel="tag">cyberduck</a>, <a href="http://blog.mfabrik.com/tag/doc/" rel="tag">doc</a>, <a href="http://blog.mfabrik.com/tag/download/" rel="tag">download</a>, <a href="http://blog.mfabrik.com/tag/drag-and-drop/" rel="tag">drag and drop</a>, <a href="http://blog.mfabrik.com/tag/export/" rel="tag">export</a>, <a href="http://blog.mfabrik.com/tag/import/" rel="tag">import</a>, <a href="http://blog.mfabrik.com/tag/mass-upload/" rel="tag">mass upload</a>, <a href="http://blog.mfabrik.com/tag/osx/" rel="tag">osx</a>, <a href="http://blog.mfabrik.com/tag/pdf/" rel="tag">pdf</a>, <a href="http://blog.mfabrik.com/tag/photos/" rel="tag">photos</a>, <a href="http://blog.mfabrik.com/tag/plone/" rel="tag">plone</a>, <a href="http://blog.mfabrik.com/tag/webdav/" rel="tag">webdav</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/18/making-an-offline-file-system-copy-of-a-plone-site-using-webdav/#respond" title="Comment on Making an offline file system copy of a Plone site using WebDAV">Leave a comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1133&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1128" class="post-1128 post type-post status-publish format-standard hentry category-plone category-python category-technology tag-answer tag-discussion tag-forum tag-irc tag-mailing-list tag-plone tag-python tag-question tag-smart-questions">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/17/rfc-simple-internet-question-asking-protocol-for-human-beings/" title="Permalink to RFC: Simple Internet Question Asking Protocol (for human beings)" rel="bookmark">RFC: Simple Internet Question Asking Protocol (for human beings)</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/17/rfc-simple-internet-question-asking-protocol-for-human-beings/" title="8:32 pm" rel="bookmark"><span class="entry-date">March 17, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>This is my  attempt version 0.1 to teach the world how one should ask questions in the simplest possible way in Internet discussion. To make it simple, I try to keep this short. This post sprouts from my frustration from the lack of people&#8217;s ability to form questions one could easily answer.</p>
<h2>Assumptions</h2>
<p>If you want to ask a question in forum, IRC (chat) or mailing list</p>
<ol>
<li>Assume people are busy</li>
<li>Assume that people want to help you, even though they are busy, since they volunteer to participate the community discussion and thus they must care about the community</li>
</ol>
<p>To make it win-win situation, you as the question maker, are responsible of making the process of asking the question and answering the question as easy as possible. <strong>Form your question in such a way that it is as easy as possible for the readers to place themselves into your situation and think how they would themselves solve the situation </strong>(Mikko&#8217;s rule of empathy)<strong>.</strong></p>
<p>The less time it takes to undestand your situation the more likely people are willing to contribute their time.</p>
<h2>Question process</h2>
<p>Thus, I propose that you always follow the simple three steps when asking a question</p>
<ol>
<li>Before asking the question tell what you <strong>already know</strong></li>
<li>Describe<strong> the problem</strong></li>
<li>Ask what you <strong>do not know</strong> yet</li>
</ol>
<p>Then <strong>wait</strong> patiently for the answer (the busy part).</p>
<h2>Pitfalls</h2>
<p>These issues often stem from the fact that the person asking the question is not familiar with text-based communication where people&#8217;s time (bandwidth) is limited and the lack of body gestures often leads to misinterpretations.</p>
<ol>
<li>Do not ask yes / no questions. You are skipping steps #1 and #3.</li>
<li>Do not saturate the bandwidth: do not repeat yourself or otherwise flood the medium. If people are busy it it does not make them un-busy by repeating yourself. You are breaking the assumption #1.</li>
<li>Do not try to pull excessive attention on you &#8211; do not try to highlight your question like &#8220;PLEASE HELP !!!!&#8221; Even if it is a matter of life and dead for you it is not for the other people who are dealing with their own matters of life and dead. You are breaking the assumption #2.</li>
</ol>
<h2>Example</h2>
<p><span style="text-decoration: line-through;">Q: Is it possible to fly me to the Moon? A: Yes</span></p>
<p>Q: I am an evil super-villain whose plan overtake the world failed.  Now I must escape. I am looking for methods to take me to the Moon or the orbit where national laws to do not apply. I am not sure should I use a shuttle or a rocket. Where could I obtain such a vehicle?</p>
<p>A: US of A just retired one reliable space shuttle what you could use. But if I were you I&#8217;d consider underwater base instead, as they will become cheaper in long run, since you can more easily produce breathable oxygen.</p>
<h2>More info</h2>
<ul>
<li><a href="http://doctormo.deviantart.com/#/d2unj89">How to ask smart questions</a></li>
<li><a href="http://plone.org/documentation/kb/ask-for-help">Plone community guidelines asking for help</a></li>
</ul>
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/plone/" title="View all posts in plone" rel="category tag">plone</a>, <a href="http://blog.mfabrik.com/category/python/" title="View all posts in python" rel="category tag">python</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/answer/" rel="tag">answer</a>, <a href="http://blog.mfabrik.com/tag/discussion/" rel="tag">discussion</a>, <a href="http://blog.mfabrik.com/tag/forum/" rel="tag">forum</a>, <a href="http://blog.mfabrik.com/tag/irc/" rel="tag">irc</a>, <a href="http://blog.mfabrik.com/tag/mailing-list/" rel="tag">mailing list</a>, <a href="http://blog.mfabrik.com/tag/plone/" rel="tag">plone</a>, <a href="http://blog.mfabrik.com/tag/python/" rel="tag">python</a>, <a href="http://blog.mfabrik.com/tag/question/" rel="tag">question</a>, <a href="http://blog.mfabrik.com/tag/smart-questions/" rel="tag">smart questions</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/17/rfc-simple-internet-question-asking-protocol-for-human-beings/#respond" title="Comment on RFC: Simple Internet Question Asking Protocol (for human beings)">Leave a comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1128&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1124" class="post-1124 post type-post status-publish format-standard hentry category-php category-technology tag-apache tag-errorlog tag-osx tag-php tag-php-ini tag-tail tag-xampp">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/16/enable-php-log-output-error_log-on-xampp-on-osx/" title="Permalink to Enable PHP log output (error_log) on XAMPP on OSX" rel="bookmark">Enable PHP log output (error_log) on XAMPP on OSX</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/16/enable-php-log-output-error_log-on-xampp-on-osx/" title="12:29 am" rel="bookmark"><span class="entry-date">March 16, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>If you are using <a href="http://www.apachefriends.org/en/xampp.html">XAMPP</a> to develop PHP software (WordPress, Joomla!) on OSX you might want to get some advanced logging output from your code. PHP provides nice <a href="http://php.net/manual/en/function.error-log.php">error_log</a>() function, but it is silent by default. Here are short instructions how to enable it and follow the log.</p>
<p>Use your favorite editor to edit php.ini file in <em>/Applications/XAMPP/etc/php.ini</em> &#8211; sudo priviledges needed, <a href="http://www.peterborgapps.com/smultron/">Smultron</a> does it out of the box.</p>
<p>Change lines:</p>
<pre>log_errors = Off</pre>
<pre>;error_log = filename</pre>
<p>To:</p>
<pre>log_errors = on</pre>
<pre>error_log = /tmp/php.log</pre>
<p>Restart Apache using <em>XAMPP controller</em> in <em>Finder -&gt; Applications</em>.</p>
<p>Now use the following UNIX command to see continuous log flow in your terminal:</p>
<pre>tail -f /tmp/php.log
</pre>
<p>See also the earlier article about <a href="http://blog.mfabrik.com/2010/12/22/local-xampp-development-and-unix-file-permissions/">XAMPP and file permissions</a>.
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/php/" title="View all posts in php" rel="category tag">php</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/apache/" rel="tag">apache</a>, <a href="http://blog.mfabrik.com/tag/error_log/" rel="tag">error_log</a>, <a href="http://blog.mfabrik.com/tag/osx/" rel="tag">osx</a>, <a href="http://blog.mfabrik.com/tag/php/" rel="tag">php</a>, <a href="http://blog.mfabrik.com/tag/php-ini/" rel="tag">php.ini</a>, <a href="http://blog.mfabrik.com/tag/tail/" rel="tag">tail</a>, <a href="http://blog.mfabrik.com/tag/xampp/" rel="tag">xampp</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/16/enable-php-log-output-error_log-on-xampp-on-osx/#respond" title="Comment on Enable PHP log output (error_log) on XAMPP on OSX">Leave a comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1124&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1119" class="post-1119 post type-post status-publish format-standard hentry category-javascript category-technology tag-chrome tag-commit-hook tag-compress tag-console tag-firefox tag-git tag-github tag-ie tag-internet-explorer tag-javascript tag-jquery tag-plone tag-preprocessor tag-python tag-safari tag-webkit">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/15/everyone-loves-and-hates-console-log/" title="Permalink to Everyone loves and hates console.log()" rel="bookmark">Everyone loves and hates console.log()</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/15/everyone-loves-and-hates-console-log/" title="4:35 pm" rel="bookmark"><span class="entry-date">March 15, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>console.log()  is the best friend of every Javascript junkie. However, the lack of it isn&#8217;t. console.log() function is only available in Webkit based browsers and with Firebug in Firefox. It&#8217;s the infamous situation that someone leaves console.log() to Javascript code, doesn&#8217;t notice its presence, commits the file and suddenly all Javascript on the production server stops working for Internet Explorer users&#8230;.</p>
<p>To tackle the lack of console.log() problem there are several approaches.</p>
<h2>Use dummy placeholder if console is missing</h2>
<p>This snippet wraps console.log (need to repeat for console.error etc.):</p>
<pre>// Ignore console on platforms where it is not available
if (typeof(window["console"]) == "undefined") { console = {}; console.log = function(a) {}; }</pre>
<p>Pros</p>
<ul>
<li>Easy</li>
</ul>
<p>Cons</p>
<ul>
<li>Need to add to every Javascript file</li>
<li>Messes with global namespace</li>
</ul>
<h2>Use module specific log function</h2>
<p>This makes your code little bit ugly, more Java like. Each Javascript module declares their own log() function which checks the existence of console.log() and outputs there if it&#8217;s present.</p>
<pre>mfabrik.log =function(x) {
 if(console.log) {
 console.log(x);
 }
}

mfabrik.log("My log messages")</pre>
<p>Pros</p>
<ul>
<li>Easy to hook other logg</li>
<li>You can disable all logging output with one if</li>
</ul>
<p>Cons</p>
<ul>
<li>Not as natural to write as console.log()</li>
<li>Need to add to every Javascript module</li>
</ul>
<h2>Preprocess Javascript files</h2>
<p>Plone (Kukit / KSS) uses this approach. All debug Javascript is  hidden behind conditional comments and it is filtered out when JS files  are bundled for the production deployment. (<a href="http://codespeak.net/svn/kukit/kss.core/trunk/kss/core/pluginregistry/_concatresource/compression/javascript.py">The preprocessing code is here in Python for those who are interested in it</a>).</p>
<pre>if (_USE_BASE2) {
 // Base2 legacy version: matchAll has to be used
 // Base2 recent version: querySelectorAll has to be used
 var _USE_BASE2_LEGACY = (typeof(base2.DOM.Document.querySelectorAll) == 'undefined');
 if (! _USE_BASE2_LEGACY) {
 ;;;     kukit.log('Using cssQuery from base2.');</pre>
<p>Pros</p>
<ul>
<li>Makes production Javascript files lighter</li>
<li>Make production Javascript files more professional &#8211; you do not deliver logging statements indented for internal purposes for your site visitors</li>
</ul>
<p>Cons</p>
<ul>
<li>Complex &#8211; preprocessing is required</li>
</ul>
<h2>Commit hooks</h2>
<p>You can use Subversion and Git commit hooks to check that committed JS files do not contain console.log. For example, Plone repositories do this for the Python statement  import pdb ; pdb.set_trace() (enforce pdb breakpoint).</p>
<p>Pros</p>
<ul>
<li>Very robust approach &#8211; you cannot create code with console.log()</li>
</ul>
<p>Cons</p>
<ul>
<li>Prevents also legitimate use of console.log()</li>
<li>Github, for example, lacks possibility to push client-side commit hooks to the repository cloners. This means that every developer must manually install commit hooks themselves. Everything manual you need to do makes the process error prone.</li>
</ul>
<h2>Other approaches?</h2>
<p>Please tell us!
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/javascript/" title="View all posts in javascript" rel="category tag">javascript</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/chrome/" rel="tag">chrome</a>, <a href="http://blog.mfabrik.com/tag/commit-hook/" rel="tag">commit hook</a>, <a href="http://blog.mfabrik.com/tag/compress/" rel="tag">compress</a>, <a href="http://blog.mfabrik.com/tag/console/" rel="tag">console</a>, <a href="http://blog.mfabrik.com/tag/firefox/" rel="tag">firefox</a>, <a href="http://blog.mfabrik.com/tag/git/" rel="tag">git</a>, <a href="http://blog.mfabrik.com/tag/github/" rel="tag">github</a>, <a href="http://blog.mfabrik.com/tag/ie/" rel="tag">ie</a>, <a href="http://blog.mfabrik.com/tag/internet-explorer/" rel="tag">internet explorer</a>, <a href="http://blog.mfabrik.com/tag/javascript/" rel="tag">javascript</a>, <a href="http://blog.mfabrik.com/tag/jquery/" rel="tag">jquery</a>, <a href="http://blog.mfabrik.com/tag/plone/" rel="tag">plone</a>, <a href="http://blog.mfabrik.com/tag/preprocessor/" rel="tag">preprocessor</a>, <a href="http://blog.mfabrik.com/tag/python/" rel="tag">python</a>, <a href="http://blog.mfabrik.com/tag/safari/" rel="tag">safari</a>, <a href="http://blog.mfabrik.com/tag/webkit/" rel="tag">webkit</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/15/everyone-loves-and-hates-console-log/#comments" title="Comment on Everyone loves and hates console.log()">1 Comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1119&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1105" class="post-1105 post type-post status-publish format-standard hentry category-appengine category-python category-technology tag-app-engine tag-data tag-download tag-mirror tag-python tag-remote-api tag-sqlite tag-upload">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/14/mirroring-app-engine-production-data-to-development-server-using-appcfg-py/" title="Permalink to Mirroring App Engine production data to development server using appcfg.py" rel="bookmark">Mirroring App Engine production data to development server using appcfg.py</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/14/mirroring-app-engine-production-data-to-development-server-using-appcfg-py/" title="12:14 am" rel="bookmark"><span class="entry-date">March 14, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>Google App Engine provides some <a href="http://code.google.com/appengine/docs/python/tools/uploadingdata.html#Setting_Up_remote_api">remote API functionality out of the box</a>. One of the remote API features  is to download data from the development server. After downloading, then you can upload the downloaded data to your development server, effectively mirroring the content of the production server to your local development server. This is very useful if you are working CMS, sites, etc. where you want to test new layout or views locally against the old data before putting them to the production.</p>
<p>First enable remote API in <em>app.yaml</em>:</p>
<pre>- url: /remote_api
  script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py
  login: admin</pre>
<blockquote><p>Note: Using <em>builtins</em> app.yaml directive didn&#8217;t work for me some reason, so I had to specify remote API URI manually.</p></blockquote>
<p>After this you should be able <a href="http://code.google.com/appengine/docs/python/tools/uploadingdata.html#Downloading_and_Uploading_All_Data">to download data</a>. Here I am using <em>appcfg.py</em> global installation on OSX. Below is the command and sample output.</p>
<pre>appcfg.py -e yourgoogleaccount@gmail.com download_data --url=http://yourappid.appspot.com/remote_api --filename=data.sqlite3
...
Downloading data records.
[INFO    ] Logging to bulkloader-log-20110313.222523
[INFO    ] Throttling transfers:
[INFO    ] Bandwidth: 250000 bytes/second
[INFO    ] HTTP connections: 8/second
[INFO    ] Entities inserted/fetched/modified: 20/second
[INFO    ] Batch Size: 10
...
[INFO    ] Have 1803 entities, 0 previously transferred
[INFO    ] 1803 entities (972883 bytes) transferred in 91.0 seconds</pre>
<p>data.sqlite3 is your production database dump in SQLite 3 binary format (used internally by the development server).</p>
<p>If you have sqlite command line tool installed you can explore around the data dump there:</p>
<pre>sqlite3 data.sqlite
SQLite version 3.7.5
Enter ".help" for instructions
Enter SQL statements terminated with a ";"

sqlite&gt; .tables
Apps                                   your-app!Model1!Entities
IdSeq                                  your-app!Model1!EntitiesByProperty
Namespaces                             your-app!Model2!Entities
bulkloader_database_signature          your-app!Model2!EntitiesByProperty
your-app!!Entities                     result
your-app!!EntitiesByProperty</pre>
<p>Now you can upload data.</p>
<blockquote><p><em>Note: </em>Even though there exists option &#8211;use_sqlite for dev_appserver.py looks like it cannot directly use the database file produced by download_data. You cannot just swap database files, you need upload the downloaded data to the development server.</p></blockquote>
<p>Start your development server:</p>
<pre>dev_appserver.py .</pre>
<p>In another terminal, go to downloaded data.sqlite folder and give the command:</p>
<pre>appcfg.py upload_data --url http://localhost:8080/remote_api --file=data.sqlite --application=yourappid</pre>
<p>It will ask you for credentials, but it seems that any username and password is accepted for the local development server.</p>
<p>Now you can login to your local development server to explore the data:</p>
<pre>http://localhost:8080/_ah/admin</pre>
<p>Ensure your data got copied over using Data Viewer:</p>
<pre>http://localhost:8080/_ah/admin</pre>
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/appengine/" title="View all posts in appengine" rel="category tag">appengine</a>, <a href="http://blog.mfabrik.com/category/python/" title="View all posts in python" rel="category tag">python</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/app-engine/" rel="tag">app engine</a>, <a href="http://blog.mfabrik.com/tag/data/" rel="tag">data</a>, <a href="http://blog.mfabrik.com/tag/download/" rel="tag">download</a>, <a href="http://blog.mfabrik.com/tag/mirror/" rel="tag">mirror</a>, <a href="http://blog.mfabrik.com/tag/python/" rel="tag">python</a>, <a href="http://blog.mfabrik.com/tag/remote-api/" rel="tag">remote api</a>, <a href="http://blog.mfabrik.com/tag/sqlite/" rel="tag">sqlite</a>, <a href="http://blog.mfabrik.com/tag/upload/" rel="tag">upload</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/14/mirroring-app-engine-production-data-to-development-server-using-appcfg-py/#respond" title="Comment on Mirroring App Engine production data to development server using appcfg.py">Leave a comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1105&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1092" class="post-1092 post type-post status-publish format-standard hentry category-appengine category-python category-technology tag-always-on tag-appengine tag-cloud tag-cloud-computing tag-cron tag-deadlineexceedederror tag-dynamic tag-gaeutilities tag-google tag-http-requiest tag-instance tag-monitoring tag-new-process tag-php tag-python tag-response-time tag-zabbix">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/11/google-app-engine-issues-with-dynamic-instances-and-deadlineexceedederrors/" title="Permalink to Google App Engine: issues with dynamic instances and DeadlineExceededErrors" rel="bookmark">Google App Engine: issues with dynamic instances and DeadlineExceededErrors</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/11/google-app-engine-issues-with-dynamic-instances-and-deadlineexceedederrors/" title="1:24 pm" rel="bookmark"><span class="entry-date">March 11, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<h2>Dynamic instances and processing time</h2>
<p>This Google App Engine feature came me as a surprise, though it makes perfect sense. Your site is slow if it has low traffic.</p>
<p>Google App Engine runs Python code on instances. By default, instances are dynamic. Instances are shutdown if they do not have enough traffic (requests per minute). Thus, when you get the individual hits to App Engine now and then, App Engine must restart your instance every time for each hit.</p>
<p>When this happens, you see the following in App Engine console logs for every request on low volume traffic:</p>
<pre>This request caused a new process to be started for your application,
and thus caused your application code to be loaded for the first time.</pre>
<p>It is not always ok to add 500 &#8211; 2000 milliseconds processing delay on the top of the normal processing time. Google&#8217;s own recommendation was that each page should be served within 200 milliseconds.</p>
<p>There are three ways to optimize this issue</p>
<ul>
<li>Use App Engine premium feature &#8220;Always on&#8221; 0,30 $ / day which keeps your instance always running</li>
<li>Use cron job or such to keep your instance alive (polling once in a minute seems to do the job)</li>
<li>Optimize your imports and split your code to several modules with light amount of imports, so that start up is fast (modules are imported only once)</li>
</ul>
<p>We are using <a href="http://www.zabbix.com/">Zabbix</a> software to monitor our sites (sidenote: I don&#8217;t recommend Zabbix as the first monitoring software choice as it is very difficult to use and has bad user experience, alienating both sysadmins and developers away from it). This is what we had before optimizations &#8211; App Engine was starting a new process for every request:</p>
<p><a href="http://blog.mfabrik.com/wp-content/uploads/2011/03/zabbix31.png"><img class="alignnone size-full wp-image-1095" title="zabbix3" src="http://blog.mfabrik.com/wp-content/uploads/2011/03/zabbix31.png" alt="" width="791" height="490" /></a></p>
<p>&#8230; and this is output we got after optimizations:</p>
<p><a href="http://blog.mfabrik.com/wp-content/uploads/2011/03/zabbix4.png"><img class="alignnone size-full wp-image-1096" title="zabbix4" src="http://blog.mfabrik.com/wp-content/uploads/2011/03/zabbix4.png" alt="" width="786" height="392" /></a></p>
<p>Here is the corresponding diagram after optimizations from App Engine dashboard itself. These processing times are without network latency. As far as I know Google does not expose the endpoints of App Engine hosting, so you don&#8217;t know from which site of the world your responses come from. By comparing this diagram to the diagram above, you can see how Internet traffic is affecting to your App Engine application.</p>
<p><a href="http://blog.mfabrik.com/wp-content/uploads/2011/03/appengine.png"><img class="alignnone size-full wp-image-1097" title="appengine" src="http://blog.mfabrik.com/wp-content/uploads/2011/03/appengine.png" alt="" width="824" height="280" /></a></p>
<h2>The PITA of dying instances</h2>
<p>For some reason, App Engine instances misbehave sometimes. This causes the HTTP requests die ungracefully.</p>
<p>Normally it is not a problem as you lost few page loads now and then. People are used to &#8220;Internet grade&#8221; service and can hit the refresh button if they have problems opening a page.</p>
<p>However if you are monitoring your site and the site gives an unnecessary alarm in the middle of the night, waking up your bastard operator from Hell, he will be very angry next morning and tell you to migrate the crappy software from unreliable Python / App Engine to more reliable PHP servers <img src='http://blog.mfabrik.com/wp-includes/images/smilies/icon_sad.gif' alt=':(' class='wp-smiley' /> </p>
<p>This is what you see in App Engine logs:</p>
<pre>A serious problem was encountered with the process that handled this request, causing it to exit.
This is likely to cause a new process to be used for the next request to your application.
If you see this message frequently, you may be throwing exceptions during the initialization of your application. (Error code 104)</pre>
<p>After digging in deeper, you see that it is a problem of instating a new object in the database, exceeding 30 seconds hard limit for processing a HTTP request:</p>
<pre>2011-03-09 05:06:20.794 / 500 30094ms 86cpu_ms 40api_cpu_ms
0kb Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;
.NET CLR 2.0.50727),gzip(gfe),gzip(gfe),gzip(gfe)

&lt;class 'google.appengine.runtime.DeadlineExceededError'&gt;:
Traceback (most recent call last):
  File "/base/data/home/apps/mfabrikkampagne/1.347249742610459821/main.py", line 494, in main
    run_wsgi_app(application)
  File "/base/python_runtime/python_lib/versions/1/google/appengine/ext/webapp/util.py", line 97, in run_wsgi_app
    run_bare_wsgi_app(add_wsgi_middleware(application))
  File "/base/python_runtime/python_lib/versions/1/google/appengine/ext/webapp/util.py", line 115, in run_bare_wsgi_app
    result = application(env, _start_response)
  File "/base/python_runtime/python_lib/versions/1/google/appengine/ext/webapp/__init__.py", line 515, in __call__
    handler.get(*groups)
  File "/base/data/home/apps/mfabrikkampagne/1.347249742610459821/main.py", line 296, in get
    try: self.session = Session()</pre>
<p>So it looks like there is a temporary hick-up in Google App Engine&#8217;s Data Store (Big Table?). In the example above the error comes from <a href="http://gaeutilities.appspot.com/">gaeutilities</a>&#8216;s Session model, but it could be any other model.</p>
<p><a href="http://code.google.com/intl/fi-FI/appengine/docs/python/runtime.html#The_Request_Timer">It is possible to catch DeadlineExceededError and temporarily work-around it, as shown in App Engine documentation</a>.</p>
<p>The best way to handle this situation is to adjust your monitoring software &#8211; Zabbix in our case. Zabbix allows you to configure triggers so that they don&#8217;t alarm on every bad item state change. Instead, you can use <em>min()</em> function and trigger the alarm after the trigger condition has failed every time during a monitoring period. Just make sure that the trigger period is at least twice long as the update interval of your web scenario: this way Zabbix can logs at least two item state changes and allows one of them to be failed one.</p>
<p>For example if</p>
<ul>
<li>Update interval of web scenario is 60 seconds</li>
<li>Trigger function must check minimal failures of 1 during 2*60 seconds + some buffer = 150 seconds.</li>
</ul>
<pre>{xxx.fi:web.test.fail[de.mfabrik.com].min(150)}=1</pre>
<p>This will allow one failed response before triggering the alarm.
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/appengine/" title="View all posts in appengine" rel="category tag">appengine</a>, <a href="http://blog.mfabrik.com/category/python/" title="View all posts in python" rel="category tag">python</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/always-on/" rel="tag">always on</a>, <a href="http://blog.mfabrik.com/tag/appengine/" rel="tag">appengine</a>, <a href="http://blog.mfabrik.com/tag/cloud/" rel="tag">cloud</a>, <a href="http://blog.mfabrik.com/tag/cloud-computing/" rel="tag">cloud computing</a>, <a href="http://blog.mfabrik.com/tag/cron/" rel="tag">cron</a>, <a href="http://blog.mfabrik.com/tag/deadlineexceedederror/" rel="tag">DeadlineExceededError</a>, <a href="http://blog.mfabrik.com/tag/dynamic/" rel="tag">dynamic</a>, <a href="http://blog.mfabrik.com/tag/gaeutilities/" rel="tag">gaeutilities</a>, <a href="http://blog.mfabrik.com/tag/google/" rel="tag">google</a>, <a href="http://blog.mfabrik.com/tag/http-requiest/" rel="tag">http requiest</a>, <a href="http://blog.mfabrik.com/tag/instance/" rel="tag">instance</a>, <a href="http://blog.mfabrik.com/tag/monitoring/" rel="tag">monitoring</a>, <a href="http://blog.mfabrik.com/tag/new-process/" rel="tag">new process</a>, <a href="http://blog.mfabrik.com/tag/php/" rel="tag">php</a>, <a href="http://blog.mfabrik.com/tag/python/" rel="tag">python</a>, <a href="http://blog.mfabrik.com/tag/response-time/" rel="tag">response time</a>, <a href="http://blog.mfabrik.com/tag/zabbix/" rel="tag">zabbix</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/11/google-app-engine-issues-with-dynamic-instances-and-deadlineexceedederrors/#comments" title="Comment on Google App Engine: issues with dynamic instances and DeadlineExceededErrors">1 Comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1092&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1089" class="post-1089 post type-post status-publish format-standard hentry category-python category-technology tag-cloud tag-ironpython tag-jython tag-microsoft tag-net tag-pypy tag-python tag-python-tools-for-visual-studio">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/10/visual-studio-and-microsoft-go-python/" title="Permalink to Visual Studio and Microsoft go Python" rel="bookmark">Visual Studio and Microsoft go Python</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/10/visual-studio-and-microsoft-go-python/" title="10:11 pm" rel="bookmark"><span class="entry-date">March 10, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p><a href="http://pytools.codeplex.com/">Microsoft Technical Computing Group has released a beta version for its Python integration for Visual Studio</a>.</p>
<p>This is, indeed, interesting development, as it clearly shows that Python has reached a new level of  programming language maturity.  Receiving this much of attention from mighty Microsoft means that Python is no longer a mere prospect member in the cabin of enterprise solutions.</p>
<p>Python Tools for Visual Studio are not focused only on Microsoft&#8217;s own .NET run-time: even Jython and PyPy are partially supported, claims the spec sheet. Looks like some kind of cloud integration is on its way &#8211; maybe Microsoft wants to challenge Google App Engine by providing even better cloud development tools?</p>
<p>Also there seems to be more information coming in PyCon&#8230;
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/python/" title="View all posts in python" rel="category tag">python</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/cloud/" rel="tag">cloud</a>, <a href="http://blog.mfabrik.com/tag/ironpython/" rel="tag">ironpython</a>, <a href="http://blog.mfabrik.com/tag/jython/" rel="tag">jython</a>, <a href="http://blog.mfabrik.com/tag/microsoft/" rel="tag">microsoft</a>, <a href="http://blog.mfabrik.com/tag/net/" rel="tag">net</a>, <a href="http://blog.mfabrik.com/tag/pypy/" rel="tag">pypy</a>, <a href="http://blog.mfabrik.com/tag/python/" rel="tag">python</a>, <a href="http://blog.mfabrik.com/tag/python-tools-for-visual-studio/" rel="tag">python tools for visual studio</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/10/visual-studio-and-microsoft-go-python/#respond" title="Comment on Visual Studio and Microsoft go Python">Leave a comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1089&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1082" class="post-1082 post type-post status-publish format-standard hentry category-technology tag-adapter tag-interface tag-page-template tag-plone tag-portlet tag-render tag-zope">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/10/how%c2%a0to-render-a-portlet-in-plone/" title="Permalink to How to render a portlet in Plone" rel="bookmark">How to render a portlet in Plone</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/10/how%c2%a0to-render-a-portlet-in-plone/" title="6:06 pm" rel="bookmark"><span class="entry-date">March 10, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>It&#8217;s easy <img src='http://blog.mfabrik.com/wp-includes/images/smilies/icon_smile.gif' alt=':)' class='wp-smiley' />  It took me only two years to figure this out.</p>
<p>Below is an example how to render a portlet in Plone programmatically. This is useful when you want to have special page layouts and you need to include portlet output from another part of the site.</p>
<ul>
<li>Portlet machinery uses Zope&#8217;s adapter pattern extensively. This allows you to override things based on the content context, HTTP request, etc.</li>
<li>A portlet is assigned to some context in some portlet manager</li>
<li>We can dig these assignments up by portlet assignment id (not user visible) or portlet type (portlet assignment interface)</li>
<li>Each portlet has its own overrideable renderer class</li>
</ul>
<p>This all makes everything flexible, though still not flexible enough for some use cases (blacklisting portlets). The downside is that accessing things through many abstraction layers and plug-in points (adaptions) is little cumbersome.</p>
<p>Here is sample code for digging up a portlet and calling its renderer:</p>
<pre>        import Acquisition
        from zope.component import getUtility, getMultiAdapter, queryMultiAdapter
        from plone.portlets.interfaces import IPortletRetriever, IPortletManager, IPortletRenderer

        def get_portlet_manager(column):
            """ Return one of default Plone portlet managers.

            @param column: "plone.leftcolumn" or "plone.rightcolumn"

            @return: plone.portlets.interfaces.IPortletManagerRenderer instance
            """
            manager = getUtility(IPortletManager, name=column)
            return manager

        def render_portlet(context, request, view, manager, interface):
            """ Render a portlet defined in external location.

            .. note ::

                Portlets can be idenfied by id (not user visible)
                or interface (portlet class). This method supports look up
                by interface and will return the first matching portlet with this interface.

            @param context: Content item reference where portlet appear

            @param manager: IPortletManagerRenderer instance

            @param view: Current view or None if not available

            @param interface: Marker interface class we use to identify the portlet. E.g. IFacebookPortlet 

            @return: Rendered portlet HTML as a string, or empty string if portlet not found
            """    

            retriever = getMultiAdapter((context, manager), IPortletRetriever)

            portlets = retriever.getPortlets()

            assignment = None

            for portlet in portlets:

                # portlet is {'category': 'context', 'assignment': , 'name': u'facebook-like-box', 'key': '/isleofback/sisalto/huvit-ja-harrasteet
                # Identify portlet by interface provided by assignment
                if interface.providedBy(portlet["assignment"]):
                    assignment = portlet["assignment"]
                    break

            if assignment is None:
                # Did not find a portlet
                return ""

            #- A special type of content provider, IPortletRenderer, knows how to render each
            #type of portlet. The IPortletRenderer should be a multi-adapter from
            #(context, request, view, portlet manager, data provider).

            renderer = queryMultiAdapter((context, request, view, manager, assignment), IPortletRenderer)

            # Make sure we have working acquisition chain
            renderer = renderer.__of__(context)

            if renderer is None:
                raise RuntimeError("No portlet renderer found for portlet assignment:" + str(assignment))

            renderer.update()
            # Does not check visibility here... force render always
            html = renderer.render()

            return html</pre>
<p>This is how you integrate it to your view class:</p>
<pre>    def render_slope_info(self):
        """ Render a portlet from another page in-line to this page 

        Does not render other portlets in the same portlet manager.
        """
        context = self.context.aq_inner
        request = self.request
        view = self

        column = "isleofback.app.frontpageportlets"

        # Our custom interface marking a portlet
        from isleofback.app.portlets.slopeinfo import ISlopeInfo

        manager = get_portlet_manager(column)

        html = render_portlet(context, request, view, manager, ISlopeInfo)
        return html</pre>
<p>&#8230;and this is how you call your view helper method from TAL page template:</p>
<pre>        &lt;div tal:replace="structure view/render_slope_info" /&gt;</pre>
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/adapter/" rel="tag">adapter</a>, <a href="http://blog.mfabrik.com/tag/interface/" rel="tag">interface</a>, <a href="http://blog.mfabrik.com/tag/page-template/" rel="tag">page template</a>, <a href="http://blog.mfabrik.com/tag/plone/" rel="tag">plone</a>, <a href="http://blog.mfabrik.com/tag/portlet/" rel="tag">portlet</a>, <a href="http://blog.mfabrik.com/tag/render/" rel="tag">render</a>, <a href="http://blog.mfabrik.com/tag/zope/" rel="tag">zope</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/10/how%c2%a0to-render-a-portlet-in-plone/#respond" title="Comment on How to render a portlet in Plone">Leave a comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1082&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1078" class="post-1078 post type-post status-publish format-standard hentry category-plone category-technology tag-ajax tag-comments tag-discussion tag-disqus tag-javascript tag-jquery tag-lazy-load tag-plone tag-static tag-varnish">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/09/lazily-load-elements-becoming-visible-using-jquery/" title="Permalink to Lazily load elements becoming visible using jQuery" rel="bookmark">Lazily load elements becoming visible using jQuery</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/09/lazily-load-elements-becoming-visible-using-jquery/" title="12:57 pm" rel="bookmark"><span class="entry-date">March 9, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>It is a useful trick to lazily load comments or such elements at the bottom of page. Some elements may be loaded only when they are scrolled visible.</p>
<ul>
<li>All users are not interested in the information and do not necessary read the article long enough to see it</li>
<li>By lazily loading such elements one can speed up the initial page load time</li>
<li>You save bandwidth</li>
<li>If you use AJAX for the dynamic elements of the page you can more easily cache your pages in static page cache (Varnish) even if the pages contain personalized bits</li>
</ul>
<p>For example, Disqus is doing this (see <a href="http://api.jquery.com/jQuery.ajax/">comments in jQuery API documentation</a>).</p>
<p>You can achieve this with <a href="http://remysharp.com/2009/01/26/element-in-view-event-plugin/">in-view plug-in for jQuery</a>.</p>
<p>Below is an example for Plone triggering <span style="text-decoration: underline;"><em>productappreciation_view</em></span> loading when our placeholder <em>div</em> tag becomes visible.</p>
<pre>...
&lt;head&gt;
  &lt;script type="text/javascript" tal:attributes="src string:${portal_url}/++resource++your.app/in-view.js"&gt;&lt;/script&gt;
&lt;/head&gt;
...
&lt;div id="comment-placefolder"&gt;

 &lt;!-- Display spinning AJAX indicator gif until our AJAX call completes --&gt;

 &lt;p&gt;
 &lt;!-- Image is in Products.CMFPlone/skins/plone_images --&gt;
 &lt;img tal:attributes="src string:${context/@@plone_portal_state/portal_url}/spinner.gif" /&gt; Loading comments
 &lt;/p&gt;

 &lt;!-- Hidden link to a view URL which will render the view containing the snippet for comments --&gt;                       
 &lt;a rel="nofollow" style="display:none" tal:attributes="href string:${context/absolute_url}/productappreciation_view" /&gt;

 &lt;script&gt;

 jq(document).ready(function() {

   // http://remysharp.com/2009/01/26/element-in-view-event-plugin/                                        
   jq("#comment-placeholder").bind("inview", function() {

     // This function is executed when the placeholder becomes visible

     // Extract URL from HTML page
     var commentURL = jq("#comment-placeholder a").attr("href");

     if (commentURL) {
     // Trigger AJAX call
       jq("#comment-placeholder").load(commentURL);
     }

   });                                     

 });     
 &lt;/script&gt;
&lt;/div&gt;</pre>
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/plone/" title="View all posts in plone" rel="category tag">plone</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/ajax/" rel="tag">ajax</a>, <a href="http://blog.mfabrik.com/tag/comments/" rel="tag">comments</a>, <a href="http://blog.mfabrik.com/tag/discussion/" rel="tag">discussion</a>, <a href="http://blog.mfabrik.com/tag/disqus/" rel="tag">disqus</a>, <a href="http://blog.mfabrik.com/tag/javascript/" rel="tag">javascript</a>, <a href="http://blog.mfabrik.com/tag/jquery/" rel="tag">jquery</a>, <a href="http://blog.mfabrik.com/tag/lazy-load/" rel="tag">lazy load</a>, <a href="http://blog.mfabrik.com/tag/plone/" rel="tag">plone</a>, <a href="http://blog.mfabrik.com/tag/static/" rel="tag">static</a>, <a href="http://blog.mfabrik.com/tag/varnish/" rel="tag">varnish</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/09/lazily-load-elements-becoming-visible-using-jquery/#comments" title="Comment on Lazily load elements becoming visible using jQuery">4 Comments</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1078&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	


			<div id="post-1073" class="post-1073 post type-post status-publish format-standard hentry category-python category-technology tag-crawl tag-find tag-follow tag-full-text tag-full-text-search tag-pdf tag-pypdf tag-python tag-scrape tag-scrapy tag-search tag-trademark tag-user-policy tag-violation tag-web-crawler">
			<h2 class="entry-title"><a href="http://blog.mfabrik.com/2011/03/08/installing-and-using-scrapy-web-crawler-to-search-text-on-multiple-sites/" title="Permalink to Installing and using Scrapy web crawler to search text on multiple sites" rel="bookmark">Installing and using Scrapy web crawler to search text on multiple sites</a></h2>

			<div class="entry-meta">
				<span class="meta-prep meta-prep-author">Posted on</span> <a href="http://blog.mfabrik.com/2011/03/08/installing-and-using-scrapy-web-crawler-to-search-text-on-multiple-sites/" title="4:37 pm" rel="bookmark"><span class="entry-date">March 8, 2011</span></a> <span class="meta-sep">by</span> <span class="author vcard"><a class="url fn n" href="http://blog.mfabrik.com/author/moo/" title="View all posts by Mikko Ohtamaa">Mikko Ohtamaa</a></span>			</div><!-- .entry-meta -->

				<div class="entry-content">
				<p>Here is a little script to use <a href="http://scrapy.org/">Scrapy</a>, a web crawling framework for Python, to search sites for references for certain texts including link content and PDFs. This is handy for cases where you need to find links violating the user policy,  trademarks which are not allowed or just to see where your template output is being used.  Our Scrapy example differs from a normal search engine as it does HTML source code level checking: you can also search for CSS classes, link targets and other elements which may be invisible for normal search engines.</p>
<p>Scrapy comes with a command-line tool and project skeleton generator. You need to generate your own Scrapy project to where you can then add your own spider classes.</p>
<p>Install Scrapy using Distribute (or setuptools):</p>
<pre>easy_install Scrapy</pre>
<p>Create project code skeleton:</p>
<pre>scrapy startproject myscraper</pre>
<p>Add your spider class skeleton by creating a file <em>myscraper/spiders/spiders.py</em>:</p>
<pre>from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor

class MySpider(CrawlSpider):
 """ Crawl through web sites you specify """

 name = "mycrawler"

 # Stay within these domains when crawling
 allowed_domains = ["www.mysite.com"]

 start_urls = [
 "http://www.mysite.com/",
 ]

 # Add our callback which will be called for every found link
 rules = [
   Rule(SgmlLinkExtractor(), follow=True)
 ]</pre>
<p>Start Scrapy to test it&#8217;s crawling properly. Run the following the top level directoty:</p>
<pre>scrapy crawl mycrawler</pre>
<p>You should see output like:</p>
<pre>2011-03-08 15:25:52+0200 [scrapy] INFO: Scrapy 0.12.0.2538 started (bot: myscraper)
2011-03-08 15:25:52+0200 [scrapy] DEBUG: Enabled extensions: TelnetConsole, SpiderContext, WebService, CoreStats, MemoryUsage, CloseSpider
2011-03-08 15:25:52+0200 [scrapy] DEBUG: Enabled scheduler middlewares: DuplicatesFilterMiddleware</pre>
<p>You can hit CTRL+C to interrupt scrapy.</p>
<p>Then let&#8217;s enhance the spider a bit to search for a blacklisted tags, with optional whitelisting in myscraper/spiders/spiders.py. We use also <a href="http://pybrary.net/pyPdf/">pyPdf</a> library to crawl inside PDF files:</p>
<pre>"""

        A sample crawler for seeking a text on sites.

"""

import StringIO

from functools import partial

from scrapy.http import Request

from scrapy.spider import BaseSpider
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor

from scrapy.item import Item

def find_all_substrings(string, sub):
    """

http://code.activestate.com/recipes/499314-find-all-indices-of-a-substring-in-a-given-string/

    """
    import re
    starts = [match.start() for match in re.finditer(re.escape(sub), string)]
    return starts

class MySpider(CrawlSpider):
    """ Crawl through web sites you specify """

    name = "mycrawler"

    # Stay within these domains when crawling
    allowed_domains = ["www.mysite.com", "www.mysite2.com", "intranet.mysite.com"]

    start_urls = [
        "http://www.mysite.com/",
        "http://www.mysite2.com/",
        "http://intranet.mysite.com/"
    ]

    # Add our callback which will be called for every found link
    rules = [
        Rule(SgmlLinkExtractor(), follow=True, callback="check_violations")
    ]

    # How many pages crawled? XXX: Was not sure if CrawlSpider is a singleton class
    crawl_count = 0

    # How many text matches we have found
    violations = 0

    def get_pdf_text(self, response):
        """ Peek inside PDF to check possible violations.

        @return: PDF content as searcable plain-text string
        """

        try:
                from pyPdf import PdfFileReader
        except ImportError:
                print "Needed: easy_install pyPdf"
                raise 

        stream = StringIO.StringIO(response.body)
        reader = PdfFileReader(stream)

        text = u""

        if reader.getDocumentInfo().title:
                # Title is optional, may be None
                text += reader.getDocumentInfo().title

        for page in reader.pages:
                # XXX: Does handle unicode properly?
                text += page.extractText()

        return text                                      

    def check_violations(self, response):
        """ Check a server response page (file) for possible violations """

        # Do some user visible status reporting
        self.__class__.crawl_count += 1

        crawl_count = self.__class__.crawl_count
        if crawl_count % 100 == 0:
                # Print some progress output
                print "Crawled %d pages" % crawl_count

        # Entries which are not allowed to appear in content.
        # These are case-sensitive
        blacklist = ["meat", "ham" ]

        # Enteries which are allowed to appear. They are usually
        # non-human visible data, like CSS classes, and may not be interesting business wise
        exceptions_after = [ "meatball",
                             "hamming",
                             "hamburg"
                     ]

        # These are predencing string where our match is allowed
        exceptions_before = [
                "bushmeat",
                "honeybaked ham"
        ]

        url = response.url

        # Check response content type to identify what kind of payload this link target is
        ct = response.headers.get("content-type", "").lower()
        if "pdf" in ct:
                # Assume a PDF file
                data = self.get_pdf_text(response)
        else:
                # Assume it's HTML
                data = response.body

        # Go through our search goals to identify any "bad" text on the page
        for tag in blacklist:

                substrings = find_all_substrings(data, tag)

                # Check entries against the exception list for "allowed" special cases
                for pos in substrings:
                        ok = False
                        for exception in exceptions_after:
                                sample = data[pos:pos+len(exception)]
                                if sample == exception:
                                        #print "Was whitelisted special case:" + sample
                                        ok = True
                                        break

                        for exception in exceptions_before:
                                sample = data[pos - len(exception) + len(tag): pos+len(tag) ]
                                #print "For %s got sample %s" % (exception, sample)
                                if sample == exception:
                                        #print "Was whitelisted special case:" + sample
                                        ok = True
                                        break
                        if not ok:
                                self.__class__.violations += 1
                                print "Violation number %d" % self.__class__.violations
                                print "URL %s" % url
                                print "Violating text:" + tag
                                print "Position:" + str(pos)
                                piece = data[pos-40:pos+40].encode("utf-8")
                                print "Sample text around position:" + piece.replace("\n", " ")
                                print "------"

        # We are not actually storing any data, return dummy item
        return Item()

    def _requests_to_follow(self, response):

        if getattr(response, "encoding", None) != None:
                # Server does not set encoding for binary files
                # Do not try to follow links in
                # binary data, as this will break Scrapy
                return CrawlSpider._requests_to_follow(self, response)
        else:
                return []</pre>
<p>Let&#8217;s tune down logging output level, so we get only relevant data in the output. In <em>myscaper/settings.py</em> add:</p>
<pre>LOG_LEVEL="INFO"</pre>
<p>Now you can run the crawler and pipe the output to a text file:</p>
<pre>scrapy crawl mycrawler &gt; violations.txt</pre>
<p>More information</p>
<ul>
<li><a href="http://doc.scrapy.org/intro/install.html#intro-install-easy">Scrapy manual</a></li>
</ul>
<p class="signature">
<a href="http://mfabrik.com/@@zoho-contact-form"><img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/mfabrik-24.png"></img></a> <a href="http://mfabrik.com/@@zoho-contact-form"">Get developers</a> <a href="http://twitter.com/mfabrik"> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml"><img valign="middle" src="http://www.feedburner.com/fb/images/pub/feed-icon16x16.png" alt="" style="vertical-align:middle;border:0"/></a> <a href="http://feeds.feedburner.com/mFabrikWebAndMobileDevelopment" rel="alternate" type="application/rss+xml">Subscribe mFabrik blog in a reader</a> <img valign="middle"  src="http://blog.mfabrik.com/wp-content/uploads/twitter-24.png"></img></a> <a href="http://twitter.com/moo9000">Follow me on Twitter</a></p>
							</div><!-- .entry-content -->
	
			<div class="entry-utility">
									<span class="cat-links">
						<span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="http://blog.mfabrik.com/category/python/" title="View all posts in python" rel="category tag">python</a>, <a href="http://blog.mfabrik.com/category/technology/" title="View all posts in technology" rel="category tag">technology</a>					</span>
					<span class="meta-sep">|</span>
													<span class="tag-links">
						<span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="http://blog.mfabrik.com/tag/crawl/" rel="tag">crawl</a>, <a href="http://blog.mfabrik.com/tag/find/" rel="tag">find</a>, <a href="http://blog.mfabrik.com/tag/follow/" rel="tag">follow</a>, <a href="http://blog.mfabrik.com/tag/full-text/" rel="tag">full text</a>, <a href="http://blog.mfabrik.com/tag/full-text-search/" rel="tag">full text search</a>, <a href="http://blog.mfabrik.com/tag/pdf/" rel="tag">pdf</a>, <a href="http://blog.mfabrik.com/tag/pypdf/" rel="tag">pypdf</a>, <a href="http://blog.mfabrik.com/tag/python/" rel="tag">python</a>, <a href="http://blog.mfabrik.com/tag/scrape/" rel="tag">scrape</a>, <a href="http://blog.mfabrik.com/tag/scrapy/" rel="tag">scrapy</a>, <a href="http://blog.mfabrik.com/tag/search/" rel="tag">search</a>, <a href="http://blog.mfabrik.com/tag/trademark/" rel="tag">trademark</a>, <a href="http://blog.mfabrik.com/tag/user-policy/" rel="tag">user policy</a>, <a href="http://blog.mfabrik.com/tag/violation/" rel="tag">violation</a>, <a href="http://blog.mfabrik.com/tag/web-crawler/" rel="tag">web crawler</a>					</span>
					<span class="meta-sep">|</span>
								<span class="comments-link"><a href="http://blog.mfabrik.com/2011/03/08/installing-and-using-scrapy-web-crawler-to-search-text-on-multiple-sites/#respond" title="Comment on Installing and using Scrapy web crawler to search text on multiple sites">Leave a comment</a></span>
				<span class="meta-sep">|</span> <span class="edit-link"><a class="post-edit-link" href="http://blog.mfabrik.com/wp-admin/post.php?post=1073&amp;action=edit" title="Edit Post">Edit</a></span>			</div><!-- .entry-utility -->
		</div><!-- #post-## -->

		
	

				<div id="nav-below" class="navigation">
					<div class="nav-previous"><a href="http://blog.mfabrik.com/page/2/" ><span class="meta-nav">&larr;</span> Older posts</a></div>
					<div class="nav-next"></div>
				</div><!-- #nav-below -->
			</div><!-- #content -->
		</div><!-- #container -->


		<div id="primary" class="widget-area" role="complementary">
			<ul class="xoxo">


			<li id="search" class="widget-container widget_search">
				<form role="search" method="get" id="searchform" action="http://blog.mfabrik.com/" >
	<div><label class="screen-reader-text" for="s">Search for:</label>
	<input type="text" value="" name="s" id="s" />
	<input type="submit" id="searchsubmit" value="Search" />
	</div>
	</form>			</li>

			<li id="archives" class="widget-container">
				<h3 class="widget-title">Archives</h3>
				<ul>
						<li><a href='http://blog.mfabrik.com/2011/03/' title='March 2011'>March 2011</a></li>
	<li><a href='http://blog.mfabrik.com/2011/02/' title='February 2011'>February 2011</a></li>
	<li><a href='http://blog.mfabrik.com/2011/01/' title='January 2011'>January 2011</a></li>
	<li><a href='http://blog.mfabrik.com/2010/12/' title='December 2010'>December 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/11/' title='November 2010'>November 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/10/' title='October 2010'>October 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/09/' title='September 2010'>September 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/08/' title='August 2010'>August 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/07/' title='July 2010'>July 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/06/' title='June 2010'>June 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/05/' title='May 2010'>May 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/04/' title='April 2010'>April 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/03/' title='March 2010'>March 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2010/01/' title='January 2010'>January 2010</a></li>
	<li><a href='http://blog.mfabrik.com/2009/12/' title='December 2009'>December 2009</a></li>
	<li><a href='http://blog.mfabrik.com/2009/11/' title='November 2009'>November 2009</a></li>
	<li><a href='http://blog.mfabrik.com/2009/10/' title='October 2009'>October 2009</a></li>
	<li><a href='http://blog.mfabrik.com/2009/09/' title='September 2009'>September 2009</a></li>
	<li><a href='http://blog.mfabrik.com/2009/08/' title='August 2009'>August 2009</a></li>
	<li><a href='http://blog.mfabrik.com/2009/07/' title='July 2009'>July 2009</a></li>
	<li><a href='http://blog.mfabrik.com/2009/02/' title='February 2009'>February 2009</a></li>
	<li><a href='http://blog.mfabrik.com/2008/12/' title='December 2008'>December 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/11/' title='November 2008'>November 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/10/' title='October 2008'>October 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/09/' title='September 2008'>September 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/08/' title='August 2008'>August 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/07/' title='July 2008'>July 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/06/' title='June 2008'>June 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/05/' title='May 2008'>May 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/04/' title='April 2008'>April 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/03/' title='March 2008'>March 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2008/02/' title='February 2008'>February 2008</a></li>
	<li><a href='http://blog.mfabrik.com/2007/11/' title='November 2007'>November 2007</a></li>
	<li><a href='http://blog.mfabrik.com/2007/10/' title='October 2007'>October 2007</a></li>
	<li><a href='http://blog.mfabrik.com/2007/09/' title='September 2007'>September 2007</a></li>
	<li><a href='http://blog.mfabrik.com/2007/08/' title='August 2007'>August 2007</a></li>
	<li><a href='http://blog.mfabrik.com/2007/07/' title='July 2007'>July 2007</a></li>
				</ul>
			</li>

			<li id="meta" class="widget-container">
				<h3 class="widget-title">Meta</h3>
				<ul>
					<li><a href="http://blog.mfabrik.com/wp-admin/">Site Admin</a></li>					<li><a href="http://blog.mfabrik.com/wp-login.php?action=logout&#038;_wpnonce=c5a860867b">Log out</a></li>
									</ul>
			</li>

					</ul>
		</div><!-- #primary .widget-area -->

	</div><!-- #main -->

	<div id="footer" role="contentinfo">
		<div id="colophon">



			<div id="site-info">
				<a href="http://blog.mfabrik.com/" title="mFabrik &#8211; mobile sites, apps, HTML5 and CMS software development" rel="home">
					mFabrik &#8211; mobile sites, apps, HTML5 and CMS software development				</a>
			</div><!-- #site-info -->

			<div id="site-generator">
								<a href="http://wordpress.org/" title="Semantic Personal Publishing Platform" rel="generator">Proudly powered by WordPress.</a>
			</div><!-- #site-generator -->

		</div><!-- #colophon -->
	</div><!-- #footer -->

</div><!-- #wrapper -->


<!-- tracker not added by Ultimate Google Analytics plugin v1.5.3: http://www.oratransplant.nl/uga -->
<!-- tracker is not added for a logged on user of this level -->


<!-- SyntaxHighlighter Start -->
	<script type="text/javascript" src="http://blog.mfabrik.com/wp-content/plugins/syntax/Scripts/shCore.js"></script>
	<script type="text/javascript" src="http://blog.mfabrik.com/wp-content/plugins/syntax/Scripts/shBrushPython.js"></script>
	<script type="text/javascript">dp.SyntaxHighlighter.HighlightAll('code');</script>
<!-- SyntaxHighlighter End -->
</body>
</html>