February 2016 - Knowledge Nation

Web scraping (web harvesting or web data extraction) is a computer software technique of extracting information from websites.
Source Wikipedia

In very simple words, scraping is extracting data from web page by processing HTML and it is extremely powerful. It is used for various purposes such as analyzing web pages, data aggregation from multiple sources, researching trends and many more.

Web scraping using php

Before starting scraping with php one should have basic knowledge of domdocument and curl.
Let’s being with an example, for website wiredskill.com, it’s a tech news aggregator, we have to do as follows.

get title and title length
get meta description
get all h1 tags
get all links from the page

Steps to follow

get page content using curl
load content in dom document
if title exists, get title and it’s length
if meta description exits, get meta description
get h1 tags and links

<?php
// url for sending http request
$url = "http://wiredskill.com";
// this will initialize a curl session
$ch = curl_init(); 
// set url for http request
curl_setopt($ch, CURLOPT_URL,$url); 
// used to get request output as string
curl_setopt($ch,CURLOPT_RETURNTRANSFER, true); 
// execute the request
$output = curl_exec($ch); 
//end of curl request
curl_close($ch);

// initialize dom document
$dom = new DOMDocument();
// load curl output in dom
@$dom->loadHtml($output);
// return a node list
// searches for all specified tag name in the dom
$title_node_list = $dom->getElementsByTagName('title');

// always a good practice to initialize variables
$title = $meta_tag = "";
$anchor_tags = $h1_headings = [];
// check if node list has length
if( $title_node_list->length )
	$title = trim($title_node_list->item(0)->nodeValue);
// get title length
$title_length = strlen($title);
var_dump($title); 
//string(84) "Home | Tech - Web Design - UI&UX - Startups - Gadgets News at one place | WiredSkill"

var_dump($title_length); // int(84)

$meta_tags = $dom->getElementsByTagName('meta');
foreach( $meta_tags as $meta_tag )
{
	// has attribute to check if the dom element has any attribute with 'name' 
	// if dom element has attribute with name then get that and
        // check if it is "description" or not
	if( $meta_tag->hasAttribute('name') && ($attribute = $meta_tag->getAttribute('name') ) && ($attribute == 'description') )
	{
		// check if meta tag has any attribute "content"
		if( $meta_tag->hasAttribute('content') )
			$meta_description = $meta_tag->getAttribute('content');
	}
}
var_dump($meta_description); 
//string(81) "WiredSkill provides tech, web design, ui-ux, startups, gadgets news at one place."

$headings_h1 = $dom->getElementsByTagName('h1');
foreach( $headings_h1 as $heading )
{
	// get text between <h1>some text</h1>
	$h1_headings[] = trim($heading->nodeValue);
	
}
print_r($h1_headings);
/*
Array
(
    [0] => WiredSkill beta
)
*/
$all_anchors_tags = $dom->getElementsByTagName('a');
foreach ($all_anchors_tags as $anchor )
{
	// check if anchor tag has attribute href
	if( $anchor->hasAttribute('href') )
	{
		// get "href" attribute of anchor tah
		$anchor_tags[] = $anchor->getAttribute('href');
	}
}

print_r($anchor_tags);
/*
Array
(
    [0] => http://wiredskill.com
    [1] => http://wiredskill.com/category/tech
    [2] => http://wiredskill.com/category/webdesign
    [3] => http://wiredskill.com/category/uiux
    .. many more 
)
*/
// always a good practice to unset variable which are no more in use 
$output = $dom = null;
unset($output,$dom);
?>

<?php

// url for sending http request

$url = "http://wiredskill.com";

// this will initialize a curl session

$ch = curl_init();

// set url for http request

curl_setopt($ch, CURLOPT_URL,$url);

// used to get request output as string

curl_setopt($ch,CURLOPT_RETURNTRANSFER, true);

// execute the request

$output = curl_exec($ch);

//end of curl request

curl_close($ch);

// initialize dom document

$dom = new DOMDocument();

// load curl output in dom

@$dom->loadHtml($output);

// return a node list

// searches for all specified tag name in the dom

$title_node_list = $dom->getElementsByTagName('title');

// always a good practice to initialize variables

$title = $meta_tag = "";

$anchor_tags = $h1_headings = [];

// check if node list has length

if( $title_node_list->length )

$title = trim($title_node_list->item(0)->nodeValue);

// get title length

$title_length = strlen($title);

var_dump($title);

//string(84) "Home | Tech - Web Design - UI&UX - Startups - Gadgets News at one place | WiredSkill"

var_dump($title_length); // int(84)

$meta_tags = $dom->getElementsByTagName('meta');

foreach( $meta_tags as $meta_tag )

{

// has attribute to check if the dom element has any attribute with 'name'

// if dom element has attribute with name then get that and

// check if it is "description" or not

if( $meta_tag->hasAttribute('name') && ($attribute = $meta_tag->getAttribute('name') ) && ($attribute == 'description') )

{

// check if meta tag has any attribute "content"

if( $meta_tag->hasAttribute('content') )

$meta_description = $meta_tag->getAttribute('content');

}

var_dump($meta_description);

//string(81) "WiredSkill provides tech, web design, ui-ux, startups, gadgets news at one place."

$headings_h1 = $dom->getElementsByTagName('h1');

foreach( $headings_h1 as $heading )

{

// get text between <h1>some text</h1>

$h1_headings[] = trim($heading->nodeValue);

}

print_r($h1_headings);

Array

(

[0] => WiredSkill beta

)

$all_anchors_tags = $dom->getElementsByTagName('a');

foreach ($all_anchors_tags as $anchor )

{

// check if anchor tag has attribute href

if( $anchor->hasAttribute('href') )

{

// get "href" attribute of anchor tah

$anchor_tags[] = $anchor->getAttribute('href');

}

print_r($anchor_tags);

Array

(

[0] => http://wiredskill.com

[1] => http://wiredskill.com/category/tech

[2] => http://wiredskill.com/category/webdesign

[3] => http://wiredskill.com/category/uiux

.. many more

)

// always a good practice to unset variable which are no more in use

$output = $dom = null;

unset($output,$dom);

All in all, web scraping with php is very easy, so do scraping just for fun because it breaches most of the website policies. Therefore, i really recommend to use it wisely after reading all terms and conditions of the website you are targeting to collect data.

This article is just to give knowledge about scraping, we are not responsible for any harm caused by using above script to any website by anyone.

What is MongoDb?

MongoDB is an open source document oriented nosql database. It is powerful, extremely fast, flexible and highly scalable database. It stores data in the form of JSON like documents BSON(Binary JSON).

Basics of Mongodb

Collection : In mongo database, data is stored in Collections. These are equivalent to tables in relational database.

Document : Document represent a record in MongoDB collection. We can store integer, string, array, objects and many more type of data as a document in collection.

Schemaless: Mongo has no schema for storing document, we can store any type of data in any cell of a row.

To install mongodb follow this article “How to install mongodb” and install andy GUI tool like MongoVue or Robomongo for better understanding.

Mongodb CRUD operations using PHP

Create records

Single Insert
Consider a scenario where we want to insert user details in “user_details” collection in “users” database.

// connecting to mongo (default localhost at port 27017)

$mongo = new mongoClient();

// to connect mongo at a specfic IP with username and password, write it as follows

$connection_string = "mongodb://mongo-username:mongo-password@ip";

$mongo = new mongoClient($connection_string);

$data_to_insert = array('name'=>'Tushar Sharma','age'=>24,'skills'=>array('php','jquery'));

try{

$mongo->users->user_details->insert($data_to_insert);

} catch(exception $e) {

// catch exception

}

That’s it, we have inserted a new record and i think you didn’t noticed one thing and that is “we don’t have to create any database or table(collection) or schema in order to create record in mongo“.

Now, if you are dealing with a single database then specify database as follows.

$mongo = new mongoClient();

$mongo_db = $mongo->selectDB("users");

$data_to_insert = array('name'=>'Tushar Sharma','age'=>24,'skills'=>array('php','jquery'));

try{

$mongo_db->user_details->insert($data_to_insert);

} catch(exception $e) {

// catch exception

}

Last insert id can be obtained using following syntax, remember that mongo create a default unique and indexed id as _id and it’s an object not string. Check the image at the end.

1	$last_insert_id = $data_to_insert['_id'];

Bulk Insert
Consider a scenario where we want to insert multiple records at once, then we write it as follows.

$mongo = new mongoClient();

$batch_insert = array(

array('name'=>'Shubham Sharma','age'=>'32',

'skills'=>array('java','struts'),'has_job'=>0),

array('name'=>'Saurabh','skills'=>array('mysql','mongo'),

'has_job'=>1),

array('name'=>'John Doe',

'skills'=>array('frontend'=>array('html','css','bootstrap'),

'backend'=>array('php','python','node js')),'has_job'=>1,'salary'=>'76k'),

);

try{

//In case of error, continueOnError prevents stopping insertion while batch

//inserting, it will left the document with error and continuing inserting

//others documents with no error.

$mongo->users->user_details->batchInsert($batch_insert,array('continueOnError' => true));

} catch(Exception $e){

echo $e->getMessage();

}

Above batch insert shows that you can insert any data type in any field.
In first case, we entered age as string instead of integer (as inserted before).
In second case, we entered a new field has_job and also remove age.
In third case, we entered a multi dimensional array and added a new field salary.

Above example shows that mongo is schemaless.

Applying Indexes
Index is applied on a collection using following query

// apply compound index as

// age descending and has_job ascending

// on user_details collection

$mongo->users->user_details->ensureIndex(array('age'=>-1,'has_job'=>1));

// apply unique index as age descending and it must be unique

$mongo->users->user_details->ensureIndex(

array('age'=>-1),

array('unique' => true)

);

Reading records
Consider a scenario where we want to read above records on the basis of various conditions, we will discuss each aspect one by one.

// get all user details from user_details collection

$user_details_cursor = $mongo->users->user_details->find();

if( $user_details_cursor->hasNext() )

{

while( $results = $user_details_cursor->getNext() )

{

// prints each record individually

print_r($results);

}

find function is used to fetch data, it returns a cursor to the data. “hasNext()” ensures that whether any data found or not. After that we have to iterate over cursor to get results, for this “getNext()” function is used.

To count number of results which is obtained from find query writes as follows

1 2	// return number of results $total_results_count = $user_details_cursor->count()

Find results with some specific conditions

// find users with age 24,
// sort results by age in descending order
// and limit results to 1
$mongo->users->user_details->find(array('age'=>24))
                            ->sort(array('age'=>-1))
                            ->limit(1);

// find users with age greater than 20 and name tushar sharma,
// sort results by age in ascending order
// and limit results to 5
$mongo->users->user_details->find(array('age'=>array('$gt'=>20),'name'=>"tushar sharma"))
                           ->sort(array('age'=>1))
                           ->limit(5);

// find users with age greater than equal to 20 and name tushar sharma,
// sort results by age in ascending order and has_job in descending order
// and limit results to 5
$mongo->users->user_details->find(array('age'=>array('$gte'=>20),'name'=>"tushar sharma"))
                           ->sort(array('age'=>1,'has_job'=>-1))
                           ->limit(5);

// find users with age lest than 30,
// sort results by age in ascending order and has_job in descending order
// and limit results to 5
$mongo->users->user_details->find(array('age'=>array('$lt'=>30)))
                           ->sort(array('age'=>1,'has_job'=>-1))
                           ->limit(5);

// find users where no age is filled,
// sort results by age in descending order
// and limit results to 1
$mongo->users->user_details->find(array('age'=>array('$exists'=>false)))
                           ->sort(array('age'=>-1))
                           ->limit(1);

// find users where age is filled,
// sort results by age in descending order
// and return 5 results after skipping first 5 results
$mongo->users->user_details->find(array('age'=>array('$exists'=>false)))
                           ->sort(array('age'=>-1))
                           ->limit(5)
                           ->skip(5);

// find users with age 24,

// sort results by age in descending order

// and limit results to 1

$mongo->users->user_details->find(array('age'=>24))

->sort(array('age'=>-1))

->limit(1);

// find users with age greater than 20 and name tushar sharma,

// sort results by age in ascending order

// and limit results to 5

$mongo->users->user_details->find(array('age'=>array('$gt'=>20),'name'=>"tushar sharma"))

->sort(array('age'=>1))

->limit(5);

// find users with age greater than equal to 20 and name tushar sharma,

// sort results by age in ascending order and has_job in descending order

// and limit results to 5

$mongo->users->user_details->find(array('age'=>array('$gte'=>20),'name'=>"tushar sharma"))

->sort(array('age'=>1,'has_job'=>-1))

->limit(5);

// find users with age lest than 30,

// sort results by age in ascending order and has_job in descending order

// and limit results to 5

$mongo->users->user_details->find(array('age'=>array('$lt'=>30)))

->sort(array('age'=>1,'has_job'=>-1))

->limit(5);

// find users where no age is filled,

// sort results by age in descending order

// and limit results to 1

$mongo->users->user_details->find(array('age'=>array('$exists'=>false)))

->sort(array('age'=>-1))

->limit(1);

// find users where age is filled,

// sort results by age in descending order

// and return 5 results after skipping first 5 results

$mongo->users->user_details->find(array('age'=>array('$exists'=>false)))

->sort(array('age'=>-1))

->limit(5)

->skip(5);

Updating records

Let’s update some of the user details which we have entered above.

// update ALL users whose age is 24 to age 25 and has-job as 1

$is_result_updated = $mongo->users->user_details->update(array('age'=>24),

array('$set'=>array('age'=>25,'has_job'=>1)),

array('multiple'=>true));

Here ‘multiple’ is used to updated all results with matching criteria, by default mongo will update only one result if we doesn’t pass third parameter in above query.

In the below image you can see data after updating records, in first case skills is an array and in second case it’s an object.

Below image shows when mongo is successfully connected then it shows connected as 1, second section shows last insert id and third section shows if update is successful then nModified and n is equal to number of rows affected.

Deleting records

Deleting records is the easiest of all, it is done using following query

1 2	// delete all users with age 24 $mongo->users->user_details->remove(array('age'=>24))

Always remember, delete by default works on all records and update works on single record.

That’s all in “Mongodb crud operations using php”.Do comment and subscribe for further updates.

Month: February 2016

Web scraping using PHP

Web scraping using php

Mongodb CRUD operations using PHP

What is MongoDb?

Basics of Mongodb

Mongodb CRUD operations using PHP

Create records

Updating records

Deleting records