Error json decode stream url

As you said, you’re receiving JSONDecodeError because b'[' and others are not valid json strings. And as long as the result is one array of JSON objects, you must obtain whole array to convert it to python obj.

So if you must use streaming request (stream=True), the only way is to read the output to a string buffer (or other kind of buffer to store temporary data) and when the transmission is finished, convert it to a JSON obj (this and other examples are written in python3):

import json
import requests

s = requests.Session()
payload = {'limit': 0}
r = s.get('https://api.coinmarketcap.com/v1/ticker', params=payload, stream=True)
r.raise_for_status()

buf = ''
for raw_rsvp in r.iter_lines(decode_unicode=True):
    buf += raw_rsvp.decode()  # bytes -> str for python3 compatibility

rsvp = json.loads(buf)

The other workaround is to catch every JSON obj from the stream and convert it to python obj, then put them one by one into a list. But it’s a bit weird and slow:

import json
import requests

s = requests.Session()
payload = {'limit': 0}
r = s.get('https://api.coinmarketcap.com/v1/ticker', params=payload, stream=True)
r.raise_for_status()

buf = ''
rsvp = []
for raw_rsvp in r.iter_lines(decode_unicode=True):
    line = raw_rsvp.decode().strip()

    if line in ('[', ']'):
        pass
    elif line == '{':
        buf += line
    elif line in ('}', '},'):
        buf += line
        if buf.endswith(','):
            buf = buf[:-1]  # trim trailing comma
        rsvp.append(json.loads(buf))
        buf = ''  # erase buffer
    else:
        buf += line

P.S.: non-streaming request will be much easier:

import requests

s = requests.Session()
payload = {'limit': 0}
r = s.get('https://api.coinmarketcap.com/v1/ticker', params=payload)
r.raise_for_status()
rsvp = r.json()

I am getting file not found error when trying to fetch image and text from remote server
Error :-

E/BitmapFactory(888): Unable to decode stream: java.io.FileNotFoundException: /http:/localhost/galerie/albums/userpics/10001/Water_lilies.jpg: open failed: ENOENT (No such file or directory)
01-03 11:06:15.159: E/BitmapFactory(888): Unable to decode stream: java.io.FileNotFoundException: /http:/localhost/galerie/albums/userpics/10001/thumb_1ce5b4759c0062faafc4a7c84c02fa99.jpg: open failed: ENOENT (No such file or directory)  

Json output of url is :-

{"cpg15x_pictures":[{"title":"Flowers","path":"http://localhost/galerie/albums/userpics/10001/Water_lilies.jpg"},{"title":"Building","path":"http://localhost/galerie/albums/userpics/10001/thumb_1ce5b4759c0062faafc4a7c84c02fa99.jpg"},{"title":"Test","path":"http://localhost/galerie/albums/userpics/10001/Sunset.jpg"},{"title":"Food","path":"http://localhost/galerie/albums/userpics/10001/f1.jpeg"}]}

I am trying to populate the list view which contains two components image view and text view. When I run the program,it populates the list with text but images are not populated.
Instead logcat is populated with error.

I referred other related questions on stackoverflow to resolve this error but I am able to fix this problem.

Reference http://www.learn2crack.com/2013/11/listview-from-json-example.html

What can be the solution ?

    response_json = self.web_fetch(url)
    response_json = response_json.decode('utf-8')
    return json.loads(response_json)


def web_fetch(self, url):
        buffer = StringIO()
        curl = pycurl.Curl()
        curl.setopt(curl.URL, url)
        curl.setopt(curl.TIMEOUT, self.timeout)
        curl.setopt(curl.WRITEFUNCTION, buffer.write)
        curl.perform()
        curl.close()
        response = buffer.getvalue().strip()
        return response

File "/Users/nab/Desktop/myenv2/lib/python2.7/site-packages/django/core/handlers/base.py" in get_response
  111.                         response = callback(request, *callback_args, **callback_kwargs)
File "/Users/nab/Desktop/pricestore/pricemodels/views.py" in view_category
  620.     apicall=api.API().search_parts(category_id= str(categoryofpart.api_id), manufacturer = manufacturer, filter = filters, start=(catpage-1)*20, limit=20, sort_by='[["mpn","asc"]]')
File "/Users/nab/Desktop/pricestore/pricemodels/api.py" in search_parts
  176.         return simplejson.loads(response_json)
File "/Users/nab/Desktop/myenv2/lib/python2.7/site-packages/simplejson/__init__.py" in loads
  455.         return _default_decoder.decode(s)
File "/Users/nab/Desktop/myenv2/lib/python2.7/site-packages/simplejson/decoder.py" in decode
  374.         obj, end = self.raw_decode(s)
File "/Users/nab/Desktop/myenv2/lib/python2.7/site-packages/simplejson/decoder.py" in raw_decode
  393.         return self.scan_once(s, idx=_w(s, idx).end())

Exception Type: JSONDecodeError at /pricemodels/2/dir/
Exception Value: Expecting value: line 1 column 1 (char 0)

11 ответов

Лучший ответ

import requests

return requests.get(url).json()

# this is the https request for data in json format
response_json = requests.get() 

# only proceed if I have a 200 response which is saved in status_code
if (response_json.status_code == 200):  
     response = response_json.json() #converting from json to dictionary using json library

json_file_path = "/path/to/example.json"

with open(json_file_path, 'r') as j:
     contents = json.loads(j.read())

contents = json.loads(json_file_path)

response = requests.get(url)
logger.info(type(response))

>>> import json
>>> x = json.loads("")
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/__init__.py", line 348, in loads
    return _default_decoder.decode(s)
  File "/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

import json

if json_string:
    x = json.loads(json_string)
else:
    // Your logic here
    x = {}

import sys, json
struct = {}
try:
  try: #try parsing to dict
    dataform = str(response_json).strip("'<>() ").replace(''', '"')
    struct = json.loads(dataform)
  except:
    print repr(resonse_json)
    print sys.exc_info()

with open("AB.json",encoding='utf-16', errors='ignore') as json_data:
     data = json.load(json_data, strict=False)

import json
struct = {}
try:
    response_json = response_json.decode('utf-8').replace('', '')
    struct = json.loads(response_json)
except:
    print('bad json: ', response_json)
return struct

Используя следующий код, я вижу, что в начале JSON есть непечатаемый символ:

$json = '{"tomorrow":"2018-09-15"}';
var_dump(json_encode($json));

Возврат:

string(37) ""ufeff{"tomorrow":"2018-09-15"}""

Строка ufeff — это Спецификация. Для его удаления воспользуйтесь следующей функцией:

function remove_utf8_bom($text){
    $bom = pack('H*','EFBBBF');
    $text = preg_replace("/^$bom/", '', $text);
    return $text;
}

Который возвращает:

string(31) ""{"tomorrow":"2018-09-15"}""

Теперь используем весь код:

function remove_utf8_bom($text)
{
    $bom = pack('H*','EFBBBF');
    $text = preg_replace("/^$bom/", '', $text);
    return $text;
}
$json = remove_utf8_bom('{"tomorrow":"2018-09-15"}');
var_dump(json_encode($json));
print_r(json_decode($json, TRUE));

Что возвращает:

string(31) ""{"tomorrow":"2018-09-15"}""
Array
(
    [tomorrow] => 2018-09-15
)

ИЗМЕНИТЬ на основе комментариев:

Измените последние строки вашего кода:

$json = remove_utf8_bom(json_encode($risposta)); // encode here
//echo json_encode($json); // don't really need this, just a test
var_dump(json_decode($json, TRUE)); // you had $json_encode here

Это возвращает ПРИМЕР:

array(1) {
  ["tomorrow"]=>
  string(10) "2018-09-15"
}

With Python, it is a really easy to retrieve data from 3rd party API services, so I made a script for this purpose. The script worked without any issue for many different API URLs, but recently, when I wanted to load the server content response from a specific API URL into json.loads method, it threw an «Unexpected UTF-8 BOM» error. In this article, we will examine what the error means and various ways to solve it.

To retrieve the data from 3rd party API service, I use this code in my Python script:

import requests
import json

url="API_ENDPOINT_URL"
r = requests.get(url)
data = json.loads(r.text)
#....do something with the data...

The above code uses requests library to read the data from URL and then it uses json.loads method to deserialize a server’s string response containing JSON data into an object.

Until this particular case, the above code worked just fine, but now I was getting the following error:

json.decoder.JSONDecodeError: Unexpected UTF-8 BOM (decode using utf-8-sig): line 1 column 1 (char 0)

The error was caused by the json.loads(r.text), so I examined the value of r.text, which had this:

ufeffn{retreived data from the api call}

The content from server’s response contained the data from the API, but it also had that strange ufeff Unicode character at the beginning. It turns out, the Unicode character with value u+feff (or xefxbbxbf in binary) is a byte order mark (BOM) character.

What is BOM

According to Wikipedia, the BOM is an optional value at the beginning of a text stream and the presence can mean different things. With UTF-8 text streams, for example, it can be used to signal that the text is encoded in UTF-8 format, while with UTF-16 & UTF-32, the presence of BOM signals the byte order of a stream.

In my case, the data was in UTF-8 and has already been received, so having that BOM character in r.text seemed unnecessary and since it was causing the json.loads method to throw the JSONDecodeError, I wanted to get rid of it.

The hint on how to solve this problem can be found in the Python error itself. It mentions «decode using utf-8-sig«, so let’s examine this next.

What is utf-8-sig?

The utf-8-sig is a Python variant of UTF-8, in which, when used in encoding, the BOM value will be written before anything else, while when used during decoding, it will skip the UTF-8 BOM character if it exists and this is exactly what I needed.

So the solution is simple. We just need to decode the data using utf-8-sig encoding, which will get rid of the BOM value. There are several ways to accomplish that.

Solution 1 — using codecs module

First, I tried to use a codecs module which is a part of a Python standard library. It contains encoders and decoders, mostly for converting text. We can use the codecs.decode() method to decode the data using utf-8-sig encoding. Something like this:

import codecs
decoded_data=codecs.decode(r.text, 'utf-8-sig')

Unfortunately, the codecs.decode method didn’t accept strings, as it threw the following error:

TypeError: decoding with ‘utf-8-sig’ codec failed (TypeError: a bytes-like object is required, not ‘str’)

Next, I tried to convert the string into a bytes object. This can be done using encode() method available for strings. If no specific encoding argument is provided, it will use the default encoding which is UTF-8 (at least on Windows):

decoded_data=codecs.decode(r.text.encode(), 'utf-8-sig')
data = json.loads(decoded_data)

The decoded_data variable finally contained data without the BOM byte order mark Unicode character and I was finally able to use it on json.loads method.

So, this worked, but I didn’t like I was using an extra module just to get rid of one Unicode BOM character.

Solution 2 — without using the codecs module

It turns out, there is a way to encode/decode strings without the need of importing codecs module. We can simply use decode() method on the return value of string.encode() method, so we can just do this:

decoded_data=r.text.encode().decode('utf-8-sig') 
data = json.loads(decoded_data)

Let’s try to simplify this further.

Solution 3 — using requests.response content property

So far, the code in this article used r.text that contains Request’s content response in a string. We can skip the encoding part all together by simply using the r.content instead as this property already contains the server content response in bytes. We then just simply use decode() method on r.content:

decoded_d=r.content.decode('utf-8-sig')
data = json.loads(decoded_data)

Solution 4 — using requests.response encoding property

We can skip the part of calling encode() and decode() methods as shown in previous examples all together and instead use the encoding property of a requests.response object. We just need to make sure we set the value before the call to r.text as shown below:

r.encoding='utf-8-sig'
data = json.loads(r.text)

Conclusion

If the json.loads() method throws an Unexpected UTF-8 BOM error, it is due to a BOM value being present in the stream or a file. In this article, we first examined what this BOM is, then we touched a bit about utf-8-sig encoding and finally, we examined 4 ways to solve this problem.

JSONDecodeError: Expecting value: line 1 column 1 (char 0) occurs while working with JSON (JavaScript Object Notation) format. You might be storing some data or trying to fetch JSON data from an API(Application Programming Interface). In this guide, we will discuss where it can creep in and how to resolve it.

JSONDecodeError means there is an incorrect JSON format being followed. For instance, the JSON data may be missing a curly bracket, have a key that does not have a value, and data not enclosed within double-quotes or some other syntactic error.

Generally, the error expecting value: line 1 column 1 (char 0) error can occur due to the following reasons.

• Non-JSON conforming quoting
• Empty JSON file
• XML output (that is, a string starting with <)

Let’s elaborate on the points stated above:

1. Non-JSON conforming quoting

JSON or JavaScript Object Notation has a similar format to that of the python dictionary datatype. A dictionary requires a key or value to be enclosed in quotes if it is a string. Similarly, the JSON standard defines that keys need not be a string. However, keys should always be enclosed within double quotes. Not following this standard can also raise an error.

2. Empty JSON file

For this example, we have taken an empty JSON file named test.py and another file named test.py, which contains the code given below. Using context manager, we open the JSON file in reading mode and load it using the load method. However, an error is thrown because the JSON file is empty.

import json

with open("test.json", "r") as file:
    data = json.load(file)

print("Data retrieved")

3. XML output (that is, a string starting with <)

The Extensible Markup Language, or simply XML, is a simple text-based format for representing structured information: documents, data, configuration, books, transactions, invoices, and much more. Similar to JSON, it is an older way of storing data. Earlier APIs used to return data in XML format; however, JSON is nowadays the preferred choice. Let’s see how we can face the expecting value: line 1 column 1 (char 0) type error in this case.

<part number="1976">
  <name>Windscreen Wiper</name>
  <description>The Windscreen wiper
    automatically removes rain
    from your windscreen, if it
    should happen to splash there.
    It has a rubber <ref part="1977">blade</ref>
    which can be ordered separately
    if you need to replace it.
  </description>
</part>

import json
with open("test.xml", "r") as file:
    data = json.load(file)
print("Data retrieved")

Let’s break down what is happening here.

• For ease of example, suppose API returns an XML format data, as shown above.
• Now, when we try to load that response from API, we will get a type error.

Resolving JSONDecodeError: Expecting value: line 1 column 1 (char 0)

1. Solution for Non-JSON conforming quoting

To resolve the type error in this case, you need to ensure that the keys and values are enclosed within the double quotes. This is necessary because it is a part of JSON syntax. It is important to realize that JSON only uses double quotes, not single quotes.

2. Solution for empty JSON file

The solution for this is self-evident in the name. To resolve the type error, in this case, make sure that the JSON file or the response from an API is not empty. If the file is empty, add content to it, and for a response from an API, use try-except to handle the error, which can be empty JSON or a 404 error, for instance.

import json
import requests


def main():
    URL = "https://api.dictionaryapi.dev/api/v2/enties/en/"
    word = input("Enter a word:")
    data = requests.get(url + word)
    data = data.text
    try:
        data_json = json.loads(data)
        print(data_json)
    except json.JSONDecodeError:
        print("Empty response")


if __name__ == "__main__":
    main()

The above code takes in a word and returns all the information related to it in a JSON format. Now in order to show how we can handle the Expecting value: line 1 column 1 (char 0) type error, we have altered the URL. entries have been changed to enties. Therefore we will get an invalid response which will not be of the JSON format. However, this is merely done to imitate the error when you might get an invalid response from an API.

• Now, if you try to run the code above, you will be prompted to enter a word. The response is saved into the data variable and later converted to a string.
• However, in the try block json.loads method, which parses JSON string to python dictionary raises an error.
• This is because the response sent by API is not of JSON format, hence can’t be parsed, resulting in JSONDecodeError.
• JSONDecodeError gets handled by the try-except block, and a “Response content is not valid JSON” gets printed as an outcome.

3. Solution for XML output(that is, a string starting with <)

To avoid type errors resulting from an XML format, we will convert it to a JSON format. However, firstly install this library.

pip install xmltodict

import json
import xmltodict

with open("test.xml", "r") as file:
    data = xmltodict.parse(file.read())
    file.close()

    json_data = json.dumps(data)
    with open("t.json", "w") as json_file:
        json_file.write(json_data)
        json_file.close()
print("Data retrieved")
print(data)

Let’s elaborate on the code above:

• We have imported two libraries, namely JSON and xmltodict.
• Using the context manager with, XML file test.xml is opened in read mode. Thereafter using the xmltodict parse method, it is converted to a dictionary type, and the file is closed.
• json.dumps() takes in a JSON object and returns a string of that object.
• Again using context manager with, a JSON file is created, XML data that was converted to a JSON string is written on it, and the file is closed.

JSONDecodeError: Expecting value: line 1 column 1 (char 0) Django

This issue is caused by the failure of Pipenv 2018.10.9. To resolve this issue, use Pipenv 2018.5.18. For more, read here.

Are you facing this error in Flask?

Many times, when you receive the data from HTTP requests, they are fetched as bytes. So, if you face JSONDecodeError, you might be dealing with a JSON in bytes. First, you need to decode the JSON by using response.decode('utf-8'). This will create a string, and then you can parse it.

FAQs

Conclusion JSONDecodeError: Expecting value: line 1 column 1 (char 0)

The following article discussed the JSONDecodeError: Expecting value: line 1 column 1 (char 0). This error is due to various decoding and formatting errors. We looked at likely situations where it can occur and how to resolve it.

Причина для JSON_ERROR_CTRL_CHAR

Причина, по которой он возвращает JSON_ERROR_CTRL_CHAR, не из-за кодировки символов (т.е. utf8 или iso). Он сталкивается с этой ошибкой, когда данные были неправильно закодированы, и результирующая строка недействительна JSON. Если вы смешиваете одиночные кавычки ' и двойные кавычки ", это может также мешать процессу кодирования, важно быть последовательным.

С учетом сказанного,
Скорее всего, он возвращает ошибку, потому что вы не отправляете какие-либо фактические данные в первую очередь. Вы отправляете пустую строку через запрос. Ну, на самом деле, вы действительно отправляете строку

<?= $strm->encodeJSON();?>

который затем становится

json_encode("<?= $strm->encodeJSON();?>");

в вашем loadmore.php.
Но в любом случае он был бы пуст, если бы вы использовали правильный тег php <?php ?>, потому что вы ничего не повторяете.

Измените

var stream= '<?= $strm->encodeJSON();?>';

для правильного тега php, а также убедитесь, что вы действительно что-то выводили, иначе stream будет пустой строкой.

var stream= '<?php echo $strm->encodeJSON(); ?>';

И он должен работать правильно.

Разное

encodeURIComponent

Как правило, при отправке данных через AJAX вы должны encodeURIComponent() его избежать каких-либо специальных символов.

data: "stream="+encodeURIComponent(stream),

$json = json_decode(urldecode($json_str), true);

.live разрознен

Функция .live устарела с 1.7, теперь вы должны использовать функцию .on для присоединения обработчиков событий. В вашем случае вам было бы лучше просто использовать сокращенную функцию .click().

ссылки должны иметь href

Теги

<a> должны иметь атрибут href, в противном случае это не будет ссылка, и вы также можете использовать <div>, если вы не собираетесь использовать его. Как минимум, вы должны иметь href="#", а затем добавить event.preventDefault().

Имена файлов

Когда вы назовете свои файлы, старайтесь держаться подальше от периодов stream.class.php, это аварии, которые ожидаются. Некоторые системы (особенно старые) не будут их интерпретировать должным образом, и вам будет трудно переименовать их все. Стандартная конвенция предлагает использовать подчеркивание или дефисы для стабильных файловых систем.

Это работает на моем конце (со всеми файлами в одной папке)

HTML default (index.php)

<?php
    include_once("stream.php");
?>

<!DOCTYPE html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js" type="text/javascript"></script>
<script type="text/javascript">
    //load more posts
    $(document).ready(function(){
        $("#active").click(function(e) {
            e.preventDefault();

            var stream = '<?php echo $strm->encodeJSON();?>'; 
            var dataString = stream;
            var request = $.ajax({  
                type: "POST",  
                url: "loadmore.php",  
                data: "stream="+encodeURIComponent(stream),
                dataType: "html",
                beforeSend:function(data){
                    $('.load-more').html('<img src="ajax-loader(1).gif" alt="Loading..." />');
                },
                success: function(html) {
                    $('#active').remove();
                    $('#stream').append(html);
                }
            });

            //ajax error reporting
            request.fail(function(jqXHR, textStatus) {
                $('#active').html(textStatus);
            });
        });
    });
</script>
</head>

<body>
    <a href="#" class='load-more' id='active'>load more posts</a>
    <div id="stream"></div>
</body>
</html>

stream.php

<?php

class Stream {
    private $limit;
    private $type;
    private $sort;
    private $offset=0;
    private $userID;
    private $catID;
    private $content = array();
    private $num_posts;

    function __construct(){
        $a = func_get_args();
        $i = func_num_args();
        if (method_exists($this,$f='__construct'.$i)) {
            call_user_func_array(array($this,$f),$a);
        }
    }

    function __construct5($limit, $type, $sort, $userID, $catID){
        $this->limit = $limit;
        $this->type = $type;
        $this->sort = $sort;
        $this->userID = $userID;
        $this->catID = $catID;
        //$this->num_posts = $this->retrieveTotal();

        //$this->setContent(); 
    }

    function __get($name) {
        if(isset($this->$name)){
            return $this->$name;
        }
    }

        public function encodeJSON(){
        foreach ($this as $key => $value){
            if($key != 'content'){
                $json->$key = $value;
            }
        }
        return json_encode($json);
    }

    public function decodeJSON($json_str){
        $json = json_decode(urldecode($json_str), true);

        foreach ($json as $key => $value){
            $this->$key = $value;
        }
    }
}

//create the object to be encoded
$strm = new Stream(5, 'toc', ' ', 1, ' ');

?>

loadmore.php

<?php

include_once('stream.php');

$strm = new Stream();

$strm->decodeJSON($_POST['stream']);

//Output a private property
echo $strm->__get("type");

?>