Skip to content

Commit

Permalink
Improving read and write
Browse files Browse the repository at this point in the history
  • Loading branch information
Fermin committed Dec 14, 2022
1 parent fa657f8 commit 09ed8a2
Show file tree
Hide file tree
Showing 15 changed files with 234 additions and 425 deletions.
53 changes: 21 additions & 32 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# Read and write simple JSON files
# Simple JSON File Reader and Writer
This package makes it easy to read and write simple JSON files. It uses generators to minimize memory usage, even when dealing with large files.

This package allows you to easily read and write simple Json files. Behind the scenes generators are used to ensure low memory usage, even when working with large files.

Here's an example on how to read a Json.
Here is an example of how to read a JSON file:

```php
use DiamondDove\SimpleJson\SimpleJsonReader;
Expand All @@ -13,15 +12,15 @@ SimpleJsonReader::create('users.json')->get()
});
```
# Installation
You can install the package via composer:
You can install the package using composer:

```
composer require diamon-dove/simple-json
```

# Usage
## Reading a JSON
Imagine you have a JSON with this content.
Suppose you have a JSON file with the following content:

```json
[
Expand All @@ -30,6 +29,8 @@ Imagine you have a JSON with this content.
]
```

To read this file in PHP, you can do the following:

```php
use DiamondDove\SimpleJson\SimpleJsonReader;

Expand All @@ -42,7 +43,6 @@ $records->each(function(array $user) {
});
```

### Reading Json file
### Working with LazyCollections
`get` will return an instance of `Illuminate\Support\LazyCollection`. This class is part of the Laravel framework. Behind the scenes generators are used, so memory usage will be low, even for large files.

Expand All @@ -63,12 +63,12 @@ return strlen($user['first_name']) > 5;
```

## Writing files
Here's how you can write a JSON file:
To write a JSON file, you can use the following code:

use DiamondDove\SimpleJson\SimpleJsonWriter;
```php
$writer = SimpleJsonWriter::create($pathToJson)
->insert([[
->push([[
'first_name' => 'John',
'last_name' => 'Doe',
],
Expand All @@ -87,32 +87,21 @@ The file at pathToJson will contain:
]
```

### Updating Row
You can also update same JSON file with these methods
```php
SimpleJsonWriter::create($pathToJson)
->where([
'first_name' => 'John',
])
->update([
'first_name' => 'John1',
'last_name' => 'Doe1',
]);
```
Without the where() method, it will update all rows

### Deleting Row

You can also use:
```php
SimpleJsonWriter::create($pathToJson)
->where([
'first_name' => 'John',
])
->delete();
SimpleJsonWriter::create($this->pathToJson)
->push([
'name' => 'Thomas',
'state' => 'Nigeria',
'age' => 22,
])
->push([
'name' => 'Luis',
'state' => 'Nigeria',
'age' => 32,
]);
```

Without the where() method, it will update all rows

# Testing
```sh
composer test
Expand Down
178 changes: 11 additions & 167 deletions src/JsonDb.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,191 +3,35 @@
namespace DiamondDove\SimpleJson;

use DiamondDove\SimpleJson\Exceptions\InvalidJsonException;
use DiamondDove\SimpleJson\Reader\JsonCollectionStreamReader;

class JsonDb
{
public $file;
private $fp;
public string $file;
protected string $dir;
private string $load;

public function __construct( $dir) {
protected bool $asArray = true;

public function __construct($dir) {
$this->dir = $dir;
}

public function check_fp_size(): int
public function toObject(): void
{
$size = 0;

if ( $this->fp ) {
$cur_size = ftell( $this->fp );
fseek( $this->fp, 0, SEEK_END );
$size = ftell( $this->fp );
fseek( $this->fp, $cur_size, SEEK_SET );
}

return $size;
$this->asArray = false;
}

public function from($file, $load = 'full'): self
public function from($file): self
{
$this->file = sprintf( '%s/%s.json', $this->dir, str_replace('.json', '', $file)); // Adding .json extension is no longer necessary
$this->load = $load;

return $this;
}

public function get(): \Closure
public function get(): \Generator
{
return function () {
if (!file_exists($this->file)) {
touch($this->file);
}

if ( 'partial' == $this->load ) {
$this->fp = fopen( $this->file, 'r+' );
if (!$this->fp) {
throw new \Exception( 'Unable to open json file' );
}

if (($size = $this->check_fp_size())) {
$content = $this->get_json_chunk($this->fp);

// We could not get the first chunk of JSON. Lets try to load everything then
if (!$content) {
$content = fread($this->fp, $size);
} else {
// We got the first chunk, we still need to put it into an array
$content = sprintf( '[%s]', $content );
}

$content = json_decode($content, true);

// Check if its arrays of jSON
if(!is_array($content) && is_object($content)) {
yield $content;
}

if(!is_array($content) && !is_object($content) ) {
throw new InvalidJsonException( 'json is invalid' );
}

foreach ($content as $row) {
yield $row;
}
}
}

$content = file_get_contents($this->file);
$content = json_decode($content, true);

// Check if its arrays of jSON
if(!is_array($content) && is_object($content)) {
yield $content;
}

if(!is_array($content) && !is_object($content) ) {
throw new InvalidJsonException( 'json is invalid' );
}

foreach ($content as $row) {
yield $row;
}
};
}

private function get_json_chunk($fp, int $start_depth = -1 ) {
$bufsz = 8192;
$start = false;
$quotes = [false, false];
$depth = 0;
$total_bytes_read = 0;
$end = false;

if (!$fp) {
return;
foreach ((new JsonCollectionStreamReader($this->file, $this->asArray))->get() as $item) {
yield $item;
}

$cur_pos = ftell( $fp );
// Get total size of file
fseek( $fp, 0, SEEK_END );
$size = ftell( $fp );
rewind( $fp );

while ( ! feof( $fp ) ) {
$buffer = fread( $fp, $bufsz );
$i = 0;

if ( false !== $buffer ) {
// So I do not have to do strlen to get read size which is linear
$read_count = ( $size > $bufsz ) ? $bufsz : $size;
$size -= $read_count;

if ( false === $start ) {
// Find first occurence of the curly bracket
$start = strpos( $buffer, '{' );
if ( false === $start ) {
$total_bytes_read += $read_count;
continue;
} else {
$i = $start+1;
$start += $total_bytes_read;
$total_bytes_read += $read_count;
}
} else {
$total_bytes_read += $read_count;
}

for ( ; isset( $buffer[ $i ] ); $i++ ) {
if ( "'" == $buffer[ $i ] && ! $quotes[1] ) {
// If quote is escaped, ignore
if ( ! empty( $buffer[ $i - 1 ] ) && '\\' == $buffer[ $i - 1 ] ) {
continue;
}

$quotes[0] = ! $quotes[0];
continue;
}

if ( '"' == $buffer[ $i ] && ! $quotes[0] ) {
// If quote is escaped, ignore
if ( ! empty( $buffer[ $i - 1 ] ) && '\\' == $buffer[ $i - 1 ] ) {
continue;
}

$quotes[1] = ! $quotes[1];
continue;
}

$is_quoted = in_array( true, $quotes, true );

if ( '{' == $buffer[ $i ] && ! $is_quoted ) {
if ( $depth == $start_depth ) {
$start = $total_bytes_read - $read_count + $i;
}
$depth++;
}

if ( '}' == $buffer[ $i ] && ! $is_quoted ) {
$depth--;
if ( $depth == $start_depth ) {
$end = $total_bytes_read - $read_count + $i +1;
break 2;
}
}
}
}
}

$chunk = '';

if ( false !== $start && false !== $end ) {
fseek( $fp, $start, SEEK_SET );
$chunk = fread( $fp, $end - $start );
}

fseek( $fp, $cur_pos, SEEK_SET );

return $chunk;
}
}
37 changes: 0 additions & 37 deletions src/JsonReader.php
View file

This file was deleted.

Loading

0 comments on commit 09ed8a2

Please sign in to comment.