Populating contents from URL to Pandas Dataframe

While reading data from HTML to pandas dataframe, sometimes the website restricts automated access which cause in occurrence of error as below :

import pandas as pd
url = 'https://finance.yahoo.com/quote/%5EHSI/components/'
dfs = pd.read_html(url)

Error:

HTTPError                                 Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_22836/1431891409.py in <module>
      1 import pandas as pd
      2 url = 'https://finance.yahoo.com/quote/%5EHSI/components/'
----> 3 dfs = pd.read_html(url)

~\AppData\Roaming\Python\Python310\site-packages\pandas\util\_decorators.py in wrapper(*args, **kwargs)
    309                     stacklevel=stacklevel,
    310                 )
--> 311             return func(*args, **kwargs)
    312 
    313         return wrapper

~\AppData\Roaming\Python\Python310\site-packages\pandas\io\html.py in read_html(io, match, flavor, header, index_col, skiprows, attrs, parse_dates, thousands, encoding, decimal, converters, na_values, keep_default_na, displayed_only)
   1111     io = stringify_path(io)
   1112 
-> 1113     return _parse(
   1114         flavor=flavor,
   1115         io=io,

~\AppData\Roaming\Python\Python310\site-packages\pandas\io\html.py in _parse(flavor, io, match, attrs, encoding, displayed_only, **kwargs)
    917 
    918         try:
--> 919             tables = p.parse_tables()
    920         except ValueError as caught:
    921             # if `io` is an io-like object, check if it's seekable

~\AppData\Roaming\Python\Python310\site-packages\pandas\io\html.py in parse_tables(self)
    237         list of parsed (header, body, footer) tuples from tables.
    238         """
--> 239         tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
    240         return (self._parse_thead_tbody_tfoot(table) for table in tables)
    241 

~\AppData\Roaming\Python\Python310\site-packages\pandas\io\html.py in _build_doc(self)
    756                     pass
    757             else:
--> 758                 raise e
    759         else:
    760             if not hasattr(r, "text_content"):

~\AppData\Roaming\Python\Python310\site-packages\pandas\io\html.py in _build_doc(self)
    737         try:
    738             if is_url(self.io):
--> 739                 with urlopen(self.io) as f:
    740                     r = parse(f, parser=parser)
    741             else:

~\AppData\Roaming\Python\Python310\site-packages\pandas\io\common.py in urlopen(*args, **kwargs)
    237     import urllib.request
    238 
--> 239     return urllib.request.urlopen(*args, **kwargs)
    240 
    241 

C:\Program Files\Python310\lib\urllib\request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    214     else:
    215         opener = _opener
--> 216     return opener.open(url, data, timeout)
    217 
    218 def install_opener(opener):

C:\Program Files\Python310\lib\urllib\request.py in open(self, fullurl, data, timeout)
    523         for processor in self.process_response.get(protocol, []):
    524             meth = getattr(processor, meth_name)
--> 525             response = meth(req, response)
    526 
    527         return response

C:\Program Files\Python310\lib\urllib\request.py in http_response(self, request, response)
    632         # request was successfully received, understood, and accepted.
    633         if not (200 <= code < 300):
--> 634             response = self.parent.error(
    635                 'http', request, response, code, msg, hdrs)
    636 

C:\Program Files\Python310\lib\urllib\request.py in error(self, proto, *args)
    561         if http_err:
    562             args = (dict, 'default', 'http_error_default') + orig_args
--> 563             return self._call_chain(*args)
    564 
    565 # XXX probably also want an abstract factory that knows when it makes

C:\Program Files\Python310\lib\urllib\request.py in _call_chain(self, chain, kind, meth_name, *args)
    494         for handler in handlers:
    495             func = getattr(handler, meth_name)
--> 496             result = func(*args)
    497             if result is not None:
    498                 return result

C:\Program Files\Python310\lib\urllib\request.py in http_error_default(self, req, fp, code, msg, hdrs)
    641 class HTTPDefaultErrorHandler(BaseHandler):
    642     def http_error_default(self, req, fp, code, msg, hdrs):
--> 643         raise HTTPError(req.full_url, code, msg, hdrs, fp)
    644 
    645 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 404: Not Found