Relaxing json data
I faced a strange problem today while bootstrapping my mongodb instances with json data sourced from a different provider.
I had a source file that had json data objects appended against each other in a large json file. This file was generated using a jq script that pretty much worked well and was already integrated into other sections of my application.
However, I now needed to bootstrap some additional data using Python. However, since the input json file did not have commas (,) after every json object and neither the data was surrounded by array brackets ... [ and ] ... my python code wasn't working right.
After a bit of research, I discovered relaxed-json which solved the issue for me.
To illustrate the formats:
INPUT FORMAT (see no commas as the end of the object ... TFB!)
{
"id": "594afecaeed9287f648ef06f",
"params_id": "247-n-kenter-ave-los-angeles-ca-90049",
"city": "Los Angeles",
"state": "CA",
"zip": "90049",
"sqft": 2191,
"lot_sqft": 12743,
"sqft_price": 1894,
"address_line_1": "247 N Kenter Ave",
"for_sale_at": "2020-01-03T06:31:18.000+00:00",
"published_at": "2020-01-03T06:31:18.987Z",
"photo": {
"id": "5cda8cd871a6092bd932ea04",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5cda8cd871a6092bd932ea04.jpg",
"description": null
},
"location": [
34.06231,
-118.48153
],
"property_type": "home",
"status": "for_sale",
"bedrooms": 3,
"bathrooms": 4.5,
"description": "Completely redone in the last six months, this Traditional style home north of Sunset in prime Brentwood is ready to move in. Gated and set back on a large lot beautifully landscaped with mature trees and featuring a huge private backyard with an immense pool. 3 bedrooms and 3.5 baths. Hardwood floors. There is a pool house and charming detached guest quarters and billiard room. All rooms have views to the front or back grounds and the home is impeccably designed for indoor/outdoor living. Easy to show.",
"current_list_price": 4150000,
"distance": null,
"badges": [],
"favorited": 2,
"hashtag": "247NKENTERAVE90049",
"mls_listing_number": "20539948",
"sold_by_ol": false,
"sold_by_ol_refund": null,
"photos": [
{
"id": "5cda8cd871a6092bd932ea04",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5cda8cd871a6092bd932ea04.jpg",
"description": null
},
{
"id": "5e0edf0c71a6090bc86cdf65",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5e0edf0c71a6090bc86cdf65.jpg",
"description": null
},
{
"id": "5e0edf8b71a6090bc86cdf76",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5e0edf8b71a6090bc86cdf76.jpg",
"description": null
}
],
"neighborhood": "Brentwood",
"solicit_showings": false,
"solicit_questions": true,
"open_houses": [],
"cobuyer_share": null,
"photo_count": 30,
"building_address": "247 N Kenter Ave, Los Angeles, CA 90049",
"next_showable_at": "2020-01-03T13:00:00.000Z",
"ol_market": "Los Angeles Area",
"ol_sub_market": "LA - Westside",
"unit_number": null,
"mls_name": "CLAW",
"listing_office": "Hurwitz James Company"
}
{
"id": "5e0e510b71a6092b35b7a512",
"params_id": "1542-w-58th-pl-los-angeles-ca-90044",
"city": "Los Angeles",
"state": "CA",
"zip": "90047",
"sqft": 858,
"lot_sqft": 3359,
"sqft_price": 582,
"address_line_1": "1542 W 58th Pl",
"for_sale_at": "2020-01-03T05:20:48.000+00:00",
"published_at": "2020-01-03T05:20:48.143Z",
"photo": {
"id": "5e0e8d7f80d4b15c159d009d",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 427,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d7f80d4b15c159d009d.jpg",
"description": null
},
"location": [
33.988247,
-118.303928
],
"property_type": "home",
"status": "for_sale",
"bedrooms": 3,
"bathrooms": 2,
"description": "Beautifully remodeled cozy single family home is now waiting for a new home owner. Completely remodeled and Move-in Ready, this property feels like home the moment you walk through the front door. The open kitchen and living room compliment the use of space perfectly. Laminate wood floors and recess lights illuminate your way throughout the entire house. This home is ideal for the first time buyer, newly wed couple or investor looking for a turnkey rental. Cozy and inviting this home is just what you've been looking for. Home is approximately a 10 to 15 mins drive from some of the LA most popular event Venues like the THE FORUM, HOLLYWOOD PARK, LA LIVE, LAX DTLA and the highly anticipated RAMS STADUIM. Come see for yourself!",
"current_list_price": 499000,
"distance": null,
"badges": [],
"favorited": null,
"hashtag": "1542W58THPL90047",
"mls_listing_number": "DW19286342",
"sold_by_ol": false,
"sold_by_ol_refund": null,
"photos": [
{
"id": "5e0e8d7f80d4b15c159d009d",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 427,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d7f80d4b15c159d009d.jpg",
"description": null
},
{
"id": "5e0e8d8080d4b15c159d00a2",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 417,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d8080d4b15c159d00a2.jpg",
"description": null
},
{
"id": "5e0e8d9f80d4b15c159d00f5",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 427,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d9f80d4b15c159d00f5.jpg",
"description": null
}
],
"neighborhood": "Harvard Park",
"solicit_showings": false,
"solicit_questions": true,
"open_houses": [],
"cobuyer_share": null,
"photo_count": 25,
"building_address": "1542 W 58th Pl, Los Angeles, CA 90047",
"next_showable_at": "2020-01-03T13:00:00.000Z",
"ol_market": "Los Angeles Area",
"ol_sub_market": "LA - South Bay",
"unit_number": null,
"mls_name": "CMRL",
"listing_office": "Realty One Group United"
}
OUTPUT FORMAT using the script below
see we have the array brackets and the objects are seperated by commas ... a fully valid json!
[
{
"id": "594afecaeed9287f648ef06f",
"params_id": "247-n-kenter-ave-los-angeles-ca-90049",
"city": "Los Angeles",
"state": "CA",
"zip": "90049",
"sqft": 2191,
"lot_sqft": 12743,
"sqft_price": 1894,
"address_line_1": "247 N Kenter Ave",
"for_sale_at": "2020-01-03T06:31:18.000+00:00",
"published_at": "2020-01-03T06:31:18.987Z",
"photo": {
"id": "5cda8cd871a6092bd932ea04",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5cda8cd871a6092bd932ea04.jpg",
"description": null
},
"location": [
34.06231,
-118.48153
],
"property_type": "home",
"status": "for_sale",
"bedrooms": 3,
"bathrooms": 4.5,
"description": "Completely redone in the last six months, this Traditional style home north of Sunset in prime Brentwood is ready to move in. Gated and set back on a large lot beautifully landscaped with mature trees and featuring a huge private backyard with an immense pool. 3 bedrooms and 3.5 baths. Hardwood floors. There is a pool house and charming detached guest quarters and billiard room. All rooms have views to the front or back grounds and the home is impeccably designed for indoor/outdoor living. Easy to show.",
"current_list_price": 4150000,
"distance": null,
"badges": [],
"favorited": 2,
"hashtag": "247NKENTERAVE90049",
"mls_listing_number": "20539948",
"sold_by_ol": false,
"sold_by_ol_refund": null,
"photos": [
{
"id": "5cda8cd871a6092bd932ea04",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5cda8cd871a6092bd932ea04.jpg",
"description": null
},
{
"id": "5e0edf0c71a6090bc86cdf65",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5e0edf0c71a6090bc86cdf65.jpg",
"description": null
},
{
"id": "5e0edf8b71a6090bc86cdf76",
"property_id": "v2/594afecaeed9287f648ef06f",
"ext": "jpg",
"width": 1080,
"height": 720,
"path": "/images/properties/v2/594afecaeed9287f648ef06f/:width/:height/5e0edf8b71a6090bc86cdf76.jpg",
"description": null
}
],
"neighborhood": "Brentwood",
"solicit_showings": false,
"solicit_questions": true,
"open_houses": [],
"cobuyer_share": null,
"photo_count": 30,
"building_address": "247 N Kenter Ave, Los Angeles, CA 90049",
"next_showable_at": "2020-01-03T13:00:00.000Z",
"ol_market": "Los Angeles Area",
"ol_sub_market": "LA - Westside",
"unit_number": null,
"mls_name": "CLAW",
"listing_office": "Hurwitz James Company"
},
{
"id": "5e0e510b71a6092b35b7a512",
"params_id": "1542-w-58th-pl-los-angeles-ca-90044",
"city": "Los Angeles",
"state": "CA",
"zip": "90047",
"sqft": 858,
"lot_sqft": 3359,
"sqft_price": 582,
"address_line_1": "1542 W 58th Pl",
"for_sale_at": "2020-01-03T05:20:48.000+00:00",
"published_at": "2020-01-03T05:20:48.143Z",
"photo": {
"id": "5e0e8d7f80d4b15c159d009d",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 427,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d7f80d4b15c159d009d.jpg",
"description": null
},
"location": [
33.988247,
-118.303928
],
"property_type": "home",
"status": "for_sale",
"bedrooms": 3,
"bathrooms": 2,
"description": "Beautifully remodeled cozy single family home is now waiting for a new home owner. Completely remodeled and Move-in Ready, this property feels like home the moment you walk through the front door. The open kitchen and living room compliment the use of space perfectly. Laminate wood floors and recess lights illuminate your way throughout the entire house. This home is ideal for the first time buyer, newly wed couple or investor looking for a turnkey rental. Cozy and inviting this home is just what you've been looking for. Home is approximately a 10 to 15 mins drive from some of the LA most popular event Venues like the THE FORUM, HOLLYWOOD PARK, LA LIVE, LAX DTLA and the highly anticipated RAMS STADUIM. Come see for yourself!",
"current_list_price": 499000,
"distance": null,
"badges": [],
"favorited": null,
"hashtag": "1542W58THPL90047",
"mls_listing_number": "DW19286342",
"sold_by_ol": false,
"sold_by_ol_refund": null,
"photos": [
{
"id": "5e0e8d7f80d4b15c159d009d",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 427,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d7f80d4b15c159d009d.jpg",
"description": null
},
{
"id": "5e0e8d8080d4b15c159d00a2",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 417,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d8080d4b15c159d00a2.jpg",
"description": null
},
{
"id": "5e0e8d9f80d4b15c159d00f5",
"property_id": "v2/5e0e510b71a6092b35b7a512",
"ext": "jpg",
"width": 640,
"height": 427,
"path": "/images/properties/v2/5e0e510b71a6092b35b7a512/:width/:height/5e0e8d9f80d4b15c159d00f5.jpg",
"description": null
}
],
"neighborhood": "Harvard Park",
"solicit_showings": false,
"solicit_questions": true,
"open_houses": [],
"cobuyer_share": null,
"photo_count": 25,
"building_address": "1542 W 58th Pl, Los Angeles, CA 90047",
"next_showable_at": "2020-01-03T13:00:00.000Z",
"ol_market": "Los Angeles Area",
"ol_sub_market": "LA - South Bay",
"unit_number": null,
"mls_name": "CMRL",
"listing_office": "Realty One Group United"
}
]
INPUT FORMAT
The below code is what worked in creating the final file that could be easily fed into my python notebook:
// this script is to create a proper json file to use python
// the jq output from diary/writing/open-listings-data-munging-wrangling.sh was not meant to create proper json
// it seems to create fairly good really-relaxed-json
// so here we create proper json
// https://www.npmjs.com/package/really-relaxed-json
var parser = require('really-relaxed-json').createParser();
const fs = require("fs");
console.log(`directory is: ${__dirname}`);
var fileContent = fs.readFileSync(__dirname+'/sample-data/open-listings/consolidated-ol-props.json', 'utf8');
fileContent = "[" + fileContent + "]";
// console.log(fileContent);
var jsonString = parser.stringToJson(fileContent)
// console.log(jsonString);
fs.writeFileSync(__dirname+'/sample-data/open-listings/consolidated-ol-props-unrelaxed.json', jsonString)
