
    'wf                       d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
m
Z
mZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&  ejN                  d      Z(dejR                  v r ee( ejT                  d             h dZ+h dZ, ejT                  dd      Z-dZ.dZ/dZ0dddddd Z1d! Zd" Z2d# Z3 G d$ d%ejh                        Z5d& Z6d' Z7d( Z8 G d) d*ejr                        Z: e jv                           G d+ d,ejx                  jz                        Z>d-d.d/Z?d0 Z@	 	 	 d3d1ZA	 	 	 	 d4d2ZBy)5z)
Google Cloud Storage pythonic interface
    N)datetime	timedelta)parse_qs)quote)urlsplit)asyn)NoOpCallback)
get_client)setup_loggingstringify_path   )__version__)get_consistency_checker)GoogleCredentials)InventoryReport)errsretry_requestvalidate_responsegcsfsGCSFS_DEBUG)loggerlevel>   private
publicreadprojectprivatebucketownerreadauthenticatedreadbucketownerfullcontrol>   r   
publicReadprojectPrivatepublicReadWriteauthenticatedReadGCSFS_DEFAULT_PROJECT i   i     P contentEncodingcacheControlcontentDispositioncontentLanguage
customTime)content_encodingcache_controlcontent_dispositioncontent_languagecustom_timec                     t        | d      S )z
    Quote characters to be safe for URL paths.
    Also quotes '/'.

    Parameters
    ----------
    s: input URL/portion

    Returns
    -------
    corrected URL
    r$   )safe)quote_urllib)ss    ^/home/panchajanya/Development/work/volatility3/venv/lib/python3.12/site-packages/gcsfs/core.pyr   r   D   s     ##    c                 Z    t         j                  |       \  }}}dj                  ||f      S )z]
    Canonicalize path to '{bucket}/{name}' form.

    Used by petastorm, do not remove.
    /)GCSFileSystem_split_pathjoin)pathbucketname_s       r4   	norm_pathr?   U   s-     $//5OFD!88VTN##r5   c                    K   | 4 d {    | j                          d {   j                         cd d d       d {    S 7 ;7 %7 	# 1 d {  7  sw Y   y xY wwN)readdecode)rs    r4   _req_to_textrE   _   sP      ) )ffh&&() ) )) ) ) )sR   A AA AAAA A	 A A	A AAAA c                       e Zd ZdZd Zy)UnclosableBytesIOz7Prevent closing BytesIO to avoid errors during retries.c                 &    | j                  d       y)z%Reset stream position for next retry.r   N)seekselfs    r4   closezUnclosableBytesIO.closeg   s    		!r5   N)__name__
__module____qualname____doc__rL    r5   r4   rG   rG   d   s
    Ar5   rG   c                  l     t        j                  dd        rt         fddD              sd    S y)z
    Resolves GCS HTTP location as http[s]://host

    Enables storage emulation for integration tests.

    Returns
    -------
    valid http location
    STORAGE_EMULATOR_HOSTNc              3   @   K   | ]  }j                  |        y wrA   )
startswith).0scheme_emulator_locations     r4   	<genexpr>z_location.<locals>.<genexpr>x   s"      
6<))&1
   )http://zhttps://r[   zhttps://storage.googleapis.com)osgetenvany)rX   s   @r4   	_locationr_   l   sI     #:DA 
@W
 
 $++=*>!?!!+r5   c              #   V   K   t        dt        |       |      D ]  }| |||z      yw)zq
    Yield evenly-sized chunks from a list.

    Implementation based on https://stackoverflow.com/a/312464.
    r   N)rangelen)lstnis      r4   _chunksrf      s4      1c#h" !a!ens   ')c                      t        |       }d|v r|j                  d       t        |      dkD  rt        dj	                  |            t        |      dk(  ry|j                         S )z<Helper to coalesce a list of object generations down to one.Nr   z?Cannot coalesce generations where more than one are defined, {}r   )setremoverb   
ValueErrorformatpop)argsgenerationss     r4   _coalesce_generationro      si    d)K{4 
;!&%
 	
 
[	Q	  r5   c                       e Zd ZdZh dZdZeZdZdZ	e
ddddddddd	d	dddddd	f fd
	Zed        Zed        Zed        Zed        Zd Zed        Zed        Zed        Zd Zd Zd Z ee      ddddd       Zd	d	ddZ ej<                  e      Zed        Z d Z!dAdZ" ej<                  e"      Z#d Z$dBdZ%	 dCd Z&d! Z'd" Z(d# Z)dDd$Z*	 	 	 	 	 	 	 dEd%Z+ ej<                  e+      Z,d& Z- ej<                  e-      Z.d' Z/d( Z0d) Z1dDd*Z2	 dFd+Z3d, Z4dGd-Z5d. Z6 ej<                  e6      Z7	 	 	 dHd/Z8 ej<                  e8      Z9dDd0Z: ej<                  e:      Z;dDd1Z<d2 Z=d3 Z>ed4        Z?dId5Z@ ej<                  e@      ZA	 	 	 	 	 dJd6ZB	 	 	 	 	 	 dKd7ZCd8 ZD	 	 	 	 	 dLd9ZE ee      ddd:d;       ZFdDd<ZG	 	 	 	 	 	 	 	 	 dMd=ZHedNd>       ZId? ZJdOd@ZK xZLS )Pr8   a  
    Connect to Google Cloud Storage.

    The following modes of authentication are supported:

    - ``token=None``, GCSFS will attempt to guess your credentials in the
      following order: gcloud CLI default, gcsfs cached token, google compute
      metadata service, anonymous.
    - ``token='google_default'``, your default gcloud credentials will be used,
      which are typically established by doing ``gcloud login`` in a terminal.
    - ``token='cache'``, credentials from previously successful gcsfs
      authentication will be used (use this after "browser" auth succeeded)
    - ``token='anon'``, no authentication is performed, and you can only
      access data which is accessible to allUsers (in this case, the project and
      access level parameters are meaningless)
    - ``token='browser'``, you get an access code with which you can
      authenticate via a specially provided URL
    - if ``token='cloud'``, we assume we are running within google compute
      or google container engine, and query the internal metadata directly for
      a token.
    - you may supply a token generated by the
      [gcloud](https://cloud.google.com/sdk/docs/)
      utility; this is either a python dictionary, the name of a file
      containing the JSON returned by logging in with the gcloud CLI tool,
      or a Credentials object. gcloud typically stores its tokens in locations
      such as
      ``~/.config/gcloud/application_default_credentials.json``,
      ``~/.config/gcloud/credentials``, or
      ``~\AppData\Roaming\gcloud\credentials``, etc.

    Specific methods, (eg. ``ls``, ``info``, ...) may return object details from GCS.
    These detailed listings include the
    [object resource](https://cloud.google.com/storage/docs/json_api/v1/objects#resource)

    GCS *does not* include  "directory" objects but instead generates
    directories by splitting
    [object names](https://cloud.google.com/storage/docs/key-terms).
    This means that, for example,
    a directory does not need to exist for an object to be created within it.
    Creating an object implicitly creates it's parent directories, and removing
    all objects from a directory implicitly deletes the empty directory.

    `GCSFileSystem` generates listing entries for these implied directories in
    listing apis with the  object properties:

        - "name" : string
            The "{bucket}/{name}" path of the dir, used in calls to
            GCSFileSystem or GCSFile.
        - "bucket" : string
            The name of the bucket containing this object.
        - "kind" : 'storage#object'
        - "size" : 0
        - "storageClass" : 'DIRECTORY'
        - type: 'directory' (fsspec compat)

    GCSFileSystem maintains a per-implied-directory cache of object listings and
    fulfills all object information and listing requests from cache. This implied, for example, that objects
    created via other processes *will not* be visible to the GCSFileSystem until the cache
    refreshed. Calls to GCSFileSystem.open and calls to GCSFile are not effected by this cache.

    In the default case the cache is never expired. This may be controlled via the ``cache_timeout``
    GCSFileSystem parameter or via explicit calls to ``GCSFileSystem.invalidate_cache``.

    Parameters
    ----------
    project : string
        project_id to work under. Note that this is not the same as, but often
        very similar to, the project name.
        This is required in order
        to list all the buckets you have access to within a project and to
        create/delete buckets, or update their access policies.
        If ``token='google_default'``, the value is overridden by the default,
        if ``token='anon'``, the value is ignored.
    access : one of {'read_only', 'read_write', 'full_control'}
        Full control implies read/write as well as modifying metadata,
        e.g., access control.
    token: None, dict or string
        (see description of authentication methods, above)
    consistency: 'none', 'size', 'md5'
        Check method when writing files. Can be overridden in open().
    cache_timeout: float, seconds
        Cache expiration time in seconds for object metadata cache.
        Set cache_timeout <= 0 for no caching, None for no cache expiration.
    secure_serialize: bool (deprecated)
    requester_pays : bool, or str default False
        Whether to use requester-pays requests. This will include your
        project ID `project` in requests as the `userProject`, and you'll be
        billed for accessing data from requester-pays buckets. Optionally,
        pass a project-id here as a string to use that as the `userProject`.
    session_kwargs: dict
        passed on to ``aiohttp.ClientSession``; can contain, for example,
        proxy settings.
    endpoint_url: str
        If given, use this URL (format protocol://host:port , *without* any
        path part) for communication. If not given, defaults to the value
        of environment variable "STORAGE_EMULATOR_HOST"; if that is not set
        either, will use the standard Google endpoint.
    default_location: str
        Default location where buckets are created, like 'US' or 'EUROPE-WEST3'.
        You can find a list of all available locations here:
        https://cloud.google.com/storage/docs/locations#available-locations
    version_aware: bool
        Whether to support object versioning. If enabled this will require the
        user to have the necessary permissions for dealing with versioned objects.
    >   	read_only
read_writefull_control   )gsgcsTrs   NnoneFc                    |r||d<   t        |   | f||d| || j                  vrt        d| j                        |t	        j
                  d       ||| _        |
| _        || _        |xs |j                  dd       | _
        |	| _        || _        d | _        || _        |xs i | _        || _        || _        |rt	        j
                  dt$               t'        |||      | _        y )Nlistings_expiry_time)asynchronousloopzaccess must be one of {}z3GCS project not set - cannot list or create bucketszVThe `check_connection` argument is deprecated and will be removed in a future release.)super__init__scopesrj   warningswarndefault_block_sizerequester_paysconsistencyrl   cache_timeoutrequests_timeouttimeout_session	_endpointsession_kwargsdefault_locationversion_awareDeprecationWarningr   credentials)rK   projectaccesstoken
block_sizer   r   secure_serializecheck_connectionr   r   rz   r   r{   r   endpoint_urlr   r   kwargs	__class__s                      r4   r}   zGCSFileSystem.__init__  s   * -:F)*	
%	
 		
 $7EE?MMOP!&0D#,&*Vfjj9OQU.V 0%,2 0*MMh"
 -WfeDr5   c                 2    | j                   xs
 t               S rA   )r   r_   rJ   s    r4   r_   zGCSFileSystem._locationA  s    ~~,,r5   c                      | j                    dS )Nz/storage/v1/r_   rJ   s    r4   basezGCSFileSystem.baseE  s    ..!..r5   c                 .    | j                   j                  S rA   r   r   rJ   s    r4   r   zGCSFileSystem.projectI  s    '''r5   c                 2   | I|F| j                         r5	 t        j                         }|j                  |j	                                y y y y # t
        $ r Y nw xY w	 t        j                  | |j                  d       y # t        j                  $ r Y y w xY w)Ng?r   )

is_runningasyncioget_running_loopcreate_taskrL   RuntimeErrorr   syncfsspecFSTimeoutError)r{   sessioncurrent_loops      r4   close_sessionzGCSFileSystem.close_sessionT  s     3 #*#;#;#=L ,,W]]_=  !4 $ IIdGMM3?,, s#   3A 	AA"B   BBc                    K   | j                   Xt        di | j                   d {   | _         t        j                  | | j
                  | j                  | j                          | j                   S 7 Kw)NrQ   )r   r
   r   weakreffinalizer   r{   rJ   s    r4   _set_sessionzGCSFileSystem._set_sessionf  sZ     == ","Ct/B/B"CCDMT4#5#5tyy$--P}} Ds   %A5A3AA5c                 `    | j                   r| j                  t        d      | j                  S )Nz+Please await _connect* before anything else)rz   r   r   rJ   s    r4   r   zGCSFileSystem.sessionl  s*    !6LMM}}r5   c                    t        |t              r|D cg c]  }| j                  |       c}S t        |      }t        | j                  t
              r| j                  fn| j                  }|D ]N  }|j                  |dz         r|t        |      dz   d  })|j                  |dz         s>|t        |      dz   d  }P |xs | j                  S c c}w )Nz://   z::   )	
isinstancelist_strip_protocolr   protocolstrrU   rb   root_marker)clsr;   pprotosr   s        r4   r   zGCSFileSystem._strip_protocolr  s    dD!489qC''*99d#$.s||S$A#,,s|| 	1Hx%/0CMA-/0D1CMA-/0		1 &s& :s   Cc                 @    | j                  |d      \  }}}|ddiS i S )NTr   r   )r9   )r   r;   r>   
generations       r4   _get_kwargs_from_urlsz#GCSFileSystem._get_kwargs_from_urls  s1    ??4t?D1j!#T**	r5   c                     |j                         D ci c]  \  }}|	|| }}}| j                  r8t        | j                  t              r| j                  }n| j                  }||d<   |S c c}}w )NuserProject)itemsr   r   r   r   )rK   r   kvparamsuser_projects         r4   _get_paramszGCSFileSystem._get_params  si    #)<<>C41aQ]!Q$CC$--s3#22#||$0F=! Ds
   
A-A-c                     i }||j                  |       d|vrdt        z   |d<   | j                  j                  |       |S )Nz
User-Agentzpython-gcsfs/)updateversionr   apply)rK   headersouts      r4   _get_headerszGCSFileSystem._get_headers  sI    JJws" /' 9Cs#
r5   c                     |j                  d      s| j                  |z   }|r& |j                  |D cg c]  }t        |       c} }|S c c}w )Nhttp)rU   r   rk   r   )rK   r;   rm   r   s       r4   _format_pathzGCSFileSystem._format_path  sH    v&99t#D4;;4 8aq 89D !9s   A)retries)r   jsondatac          	      6  K   | j                          d {    t        |d      r|j                  d       | j                  j	                  || j                  ||      | j                  |      || j                  |      || j                        4 d {   }|j                  }	|j                  }|j                  }
|j                          d {   }t        |	|||       |	||
|fcd d d       d {    S 7 7 g7 -7 # 1 d {  7  sw Y   y xY ww)NrI   r   )methodurlr   r   r   r   r   )r   hasattrrI   r   requestr   r   r   r   statusr   request_inforB   r   )rK   r   r;   r   r   r   rm   r   rD   r   infocontentss               r4   _requestzGCSFileSystem._request  s     !!!4 IIaL<<''!!$-##F+%%g.)) ( 
 	3 	3 XXFiiG>>DVVX~Hfhd;7D(2	3 	3 	3 	"	3 &	3 	3 	3 	3sj   DC<A?DC>D8DD D*D6D7D>D DDD
DDD)json_outinfo_outc                  K   t         j                  |j                          d| d| d|j                  d               | j                  ||g|i | d {   \  }}}	}
|rt        j                  |
      S |r|	S ||
fS 7 (w)Nz: z, r   )r   debuguppergetr   r   loads)rK   r   r;   r   r   rm   r   r   r   r   r   s              r4   _callzGCSFileSystem._call  s      	'r$r$r&**Y:O9PQR0=D1
1
#)1
 +
'x ::h''KH$$+
s   ABB)Bc                     t        j                  | j                  | j                  | j                        D cg c]  }|d   	 c}S c c}w )z)Return list of available project buckets.r   r=   )r   r   r{   _list_bucketsr   )rK   bs     r4   bucketszGCSFileSystem.buckets  sC    
 YYtyy$*<*<dllS
 fI
 	
 
s   Ac                 f   t        |      }t        |j                  dd            |d<   t        j                  ||d         |d<   d|d<   d|v r| j                  |d         |d<   d|v r| j                  |d         |d	<   d
|v sd|v r(|j                  d
      |d
<   |j                  d      |d<   |S )aW  Process object resource into gcsfs object information format.

        Process GCS object resource via type casting and attribute updates to
        the cache-able gcsfs object information format. Returns an updated copy
        of the object resource.

        (See https://cloud.google.com/storage/docs/json_api/v1/objects#resource)
        sizer   r=   filetypeupdatedmtimetimeCreatedctimer   metageneration)dictintr   	posixpathr:   _parse_timestamp)rK   r<   object_metadataresults       r4   _process_objectzGCSFileSystem._process_object  s     o&_00;<v"0GHvv '"33OI4NOF7OO+"33OM4RSF7O?*.>/.Q#2#6#6|#DF< '6':':;K'LF#$r5   c                 Z   K   dd|ii}| j                  dd| |       d {    y 7 w)NbillingrequesterPaysPATCHb/)r   )r   )rK   r;   stater   s       r4   _make_bucket_requester_paysz)GCSFileSystem._make_bucket_requester_pays  s5      OU34jjBtf+Dj999s   !+)+c           
        K   | j                  |      \  }}}| j                  |      }|rQdj                  ||f      }|D ].  }|d   dk(  s|d   |k(  s|r|j                  d      |k(  s,|c S  t	        |      |st	        |      d}	 | j                  dd||d	|
       d{   }| j                  ||      S 7 # t        $ r}	t        |	      j                  d      s | j                  dd|d	||sdnd|rdnd       d{  7  }
|
j                  dg       D ]%  }|d   |k(  s|r|j                  d      |k(  s#|} n |t	        |      Y d}	~	d}	~	ww xY ww)z,Return object information at the given path.r7   r   r   r=   r   NGET	b/{}/o/{}T)r   r   	Forbiddenb/{}/or   true)r   prefix
maxResultsversionsr   )

split_path_ls_from_cacher:   r   FileNotFoundErrorr   OSErrorr   rU   r   )rK   r;   r<   keyr   listingr=   file_detailsreserespitems               r4   _get_objectzGCSFileSystem._get_object  s    "&//$"7Z %%d+88VSM*D ' . (F2$V,4'<+;+;L+IZ+W''. (-- $D))	.

{FC$: #  C. ##FC00/  	.q6$$[1$.1D#-4 $   D "- <3&"dhh|&<
&JC {'-- %	.sl   A	E E E ,E B? &B='B? +E =B? ?	E>ED	"E*EEE EE c           	      P  K   | j                        \  }}}j                  d      |j                  dd       }|s0	 | j                        }	|	xr t	        fd|	D              }
|	r|
s|	S  | j                  f||d| d {   \  }}|D cg c]  }||dz   |j                  d      z   dddd! }}||z   s|r| j                         d {   gS g S ||z   }|xr |j                  d	      }s|s|| j                  <   |S # t
        $ r |r Y w xY w7 c c}w 7 Ow)
Nr7   inventory_report_infoc              3   `   K   | ]%  }|d    j                  d      k(  xr |d   dk(   ' yw)r=   r7   r   	directoryN)rstrip)rV   cr;   s     r4   rY   z.GCSFileSystem._list_objects.<locals>.<genexpr>4  s>      . fI$$S)T1Nai;6NN.s   +.r  r	  r   	DIRECTORYr  r<   r=   r   storageClassr   use_snapshot_listing)
r
  r  r   r  r^   r  _do_list_objectsstripr  dircache)rK   r;   r  r	  r   r<   r  r   r  clisting
hassubdirsr   prefixes
pseudodirsr   r   s    `              r4   _list_objectszGCSFileSystem._list_objects(  s    "&//$"7Z{{3 !'

+BD I %..t4% # .%. +
 J#O !6 5 5!
(!
6<!
 
x #	
  !v||C'88 +#	

 	
 
""..t4455	5 4  
9N9R9R":
 2"%DMM$
E %  

	
 5sR   ;D&.D -D&DD&$D7D&D$6D&DD&DD&D&c           	        K   | j                  |      \  }}}	|sdn|j                  d      dz   }| | xs d}t        |xs |	      }d}
|j                  dd      }|Yt	        j
                  | ||       d{   \  }}|j                  d      }|r||fS | j                  ||||||
       d{   S | j                  ||dd|||
	       d{   S 7 ]7 &7 w)
z5Object listing for the given {bucket}/{prefix}/ path.r$   r7   Ni  r  )gcs_file_systemr  r  r   )r   r<   	delimiterr  r	  	page_sizer<   r+  start_offset
end_offsetr  r	  r,  )r
  r  boolr   r   fetch_snapshot_concurrent_list_objects_helper_sequential_list_objects_helper)rK   r;   max_resultsr+  r  r	  r   r<   _pathr   default_page_sizer  r   r&  r   s                  r4   r!  zGCSFileSystem._do_list_objects^  s5     %)OOD$9!zU\\#%6%<76(#+t.J/ ! !'

+BD I !,$3$B$B $&;% OE8 $9#<#<=S#T  $h& ==#!+ >    ==#!!+ >   5s6   A8C:C;8C3C4CCCCCc                   K   t        |D cg c]  }|d   	 c}      }t        |      |z  dz   }	t        |      |	z  }
g }g }t        |	      D ]e  }||
z  }||	dz
  k(  rt        |      }n||
z   }|dk(  rd}n||   }|t        |      k(  rd}n||   }|j                  |       |j                  |       g t	        j
                  t        dt        |            D cg c]   }| j                  ||||   ||   |||      " c}  d{   }g }g }t        t        |            D ],  }||   \  }}|j                  |       |j                  |       . ||fS c c}w c c}w 7 Yw)z
        Lists objects using coroutines, using the object names from the inventory
        report to split up the ranges.
        r=   r   r   Nr-  )sortedrb   ra   appendr   gatherr3  extend)rK   r   r<   r+  r  r	  r,  r  snapshot_object_namesnum_coroutinesnum_objects_per_coroutinestart_offsetsend_offsetsre   range_start	range_endprefix_start
prefix_endresultsr&  items_from_processprefixes_from_processs                         r4   r2  z-GCSFileSystem._concurrent_list_objects_helper  s     !''GV'G H 23y@1D$'(=$>.$P! ~& 	+A77KNQ&& 56	'*CC	a#4[AC 566!
29=
  .z*%	+*   q#m"45  44!'!.q!1*1~!%' 5 
 
  s7|$ 	3A8?
5 5LL+,OO12	3
 hq (HB
s)   
E#ECE#%E E#E!AE#c                 x  K   g }g }	| j                  dd||||||d|rdnd
       d{   }
|j                  |
j                  dg              |	j                  |
j                  dg              |
j                  d	d      }|| j                  dd|||||||d|rdnd
       d{   }
|
d   dk(  sJ |j                  |
j                  dg              |	j                  |
j                  dg              |
j                  d	d      }||	D cg c]  }| j                  ||       }	}|	|fS 7 7 c c}w w)zP
        Sequential list objects within the start and end offset range.
        r  r  Tr  N)r+  r  startOffset	endOffsetr  r   r	  r&  r   nextPageToken)r+  r  rI  rJ  r  	pageTokenr   r	  kindzstorage#objects)r   r;  r   r   )rK   r<   r+  r.  r/  r  r	  r,  r&  r   pagenext_page_tokenre   s                r4   r3  z-GCSFileSystem._sequential_list_objects_helper  si     ZZ$  'VT   
 
 	R01TXXgr*+((?D9)#($$)#+ $  D <#4444OODHHZ45LL'2./"hh=O% )( ;@@Q%%fa0@@hO
$& As;   'D:D0A=D:'D3(A$D:D:D5*D:3D:5D:c                 R  K   d| j                   vrg }| j                  dd| j                  d       d{   }|d   dk(  sJ |j                  |j	                  d	g              |j	                  d
d      }|h| j                  dd| j                  |d       d{   }|d   dk(  sJ |j                  |j	                  d	g              |j	                  d
d      }|h|D cg c]  }i ||d   dz   ddd }}|| j                   d<   |S | j                   d   S 7 7 c c}w w)z5Return list of all buckets under the current project.r$   r  r   T)r   r   NrM  zstorage#bucketsr   rK  )r   rL  r   r=   r7   r   r  r=   r   r   )r#  r   r   r;  r   )rK   r   rN  rO  re   r   s         r4   r   zGCSFileSystem._list_buckets  sN    T]]"EE3tTTD<#4444LL'2./"hh=O!-!ZZ LL-! (   F|'8888TXXgr23"&((?D"A "-  O1Nai#oq+NG  !(DMM"N}}R  3 Us;   3D'DA&D'D AD'!D'%D"<#D' D'"D'c                    |0t         j                  d       | j                  j                          y| j	                  |      j                  d      }|r1| j                  j                  |d       | j                  |      }|r0yy)z
        Invalidate listing cache for given path, it is reloaded on next use.

        Parameters
        ----------
        path: string or None
            If None, clear all listings cached else listings at or under given
            path.
        Nzinvalidate_cache clearing cacher7   )r   r   r#  clearr   r  rl   _parentrK   r;   s     r4   invalidate_cachezGCSFileSystem.invalidate_cache/  sk     <LL:;MM!''-44S9D!!$-||D) r5   c	           
         K   | j                  |      \  }
}}|
dv rt        d      d|v r|r| j                  |
       d{   ryd|v r'|s%| j                  |
       d{   ryt        |
      d|
i}|xs | j                  }|r||d<   |rddi|d	<   |r	||d
<   d}d}|	r|j                  |	       | j                  dd|| j                  |t        |      j                         |d       d{    | j                  |
       y7 7 7 w)a  
        New bucket

        If path is more than just a bucket, will create bucket if create_parents=True;
        otherwise is a noop. If create_parents is False and bucket does not exist,
        will produce FileNotFoundError.

        Parameters
        ----------
        path: str
            bucket name. If contains '/' (i.e., looks like subdir), will
            have no effect because GCS doesn't have real directories.
        acl: string, one of bACLs
            access for the bucket itself. See:
            https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
        default_acl: str, one of ACLs
            default ACL for objects created in this bucket
        location: Optional[str]
            Location where buckets are created, like 'US' or 'EUROPE-WEST3'.
            If not provided, defaults to `self.default_location`.
            You can find a list of all available locations here:
            https://cloud.google.com/storage/docs/locations#available-locations
        create_parents: bool
            If True, creates the bucket in question, if it doesn't already exist
        enable_versioning: bool
            If True, creates the bucket in question with object versioning
            enabled.
        enable_object_retention: bool
            If True, creates the bucket in question with object retention
            permanently enabled.
        iam_configuration: dict
            If provided, sets the IAM policy for the bucket. This argument
            allows setting properties such as `{publicAccessPrevention: "enforced"}`
            and `{"uniformBucketLevelAccess": {"enabled": True}}`. If passed, `acl`
            and `default_acl` are explicitly ignored.
        **kwargs
            Additional parameters passed to the API call request body. See:
            https://cloud.google.com/storage/docs/json_api/v1/buckets/insert#request-body
            for all possible options. Pass nested parameters as dictionaries, e.g.:
            `{"autoclass": {"enabled": True}}`
        )r$   r7   zCannot create root bucketr7   Nr=   locationenabledT
versioningiamConfigurationPOSTr   )r   r;   predefinedAclr   predefinedDefaultObjectAclenableObjectRetentionr   r   )r
  rj   _existsr  r   r   r   r   r   lowerrV  )rK   r;   acldefault_aclrX  create_parentsenable_versioningenable_object_retentioniam_configurationr   r<   objectr   	json_datas                 r4   _mkdirzGCSFileSystem._mkdirC  sB    j &*__T%:"
Y899$;>DLL4H.H.H$;~\\&)))#F++V$	4t44$,Ij!'0$&7Il#,=I()CKV$jjLL'2"%&=">"D"D"F  	
 		
 		
 	f%? /I *"		
s6   ?DDD!D
"BD1D2D
DDc                    K   |j                  d      }d|v ry| j                  dd|z   d       d{    | j                  |       | j                  d       y7 'w)zDelete an empty bucket

        Parameters
        ----------
        bucket: str
            bucket name. If contains '/' (i.e., looks like subdir), will
            have no effect because GCS doesn't have real directories.
        r7   NDELETEr   Fr   r$   )r  r   rV  )rK   r<   s     r4   _rmdirzGCSFileSystem._rmdir  s^      s#&=jj4&=5jAAAf%b! 	Bs   1AA(Ac                 *    | j                  |      d   S )Nr   r   rU  s     r4   modifiedzGCSFileSystem.modified      yyw''r5   c                 *    | j                  |      d   S )Nr   rp  rU  s     r4   createdzGCSFileSystem.created  rr  r5   c           	          |j                  d      sJ |d d }|ddt        |j                  dd      d         z
  z  z   }t        j                  |dz         S )NZ0rt   .r   z+00:00)endswithrb   rsplitr   fromisoformat)rK   	timestamps     r4   r   zGCSFileSystem._parse_timestamp  sb    !!#&&&crN	q3y/?/?Q/G/J+K'K LL	%%i(&:;;r5   c                   K   | j                  |      }d|vr5	 | j                  dd| d       d{   }|j                  dd	       |S | j                  |      }| j                  |      }| j                  |      \  }}	}
t        ||
      }|rLdj                  ||	f      }|D ]4  }|d   j                  d      |k(  s|r|j                  d      |k(  s2|c S  | j                  |      r||j                  d      ddddS 	 | j                  |       d{   }|d   s|d   j                  d      s|S 	 d|d<    | j                  |fi | d{   }|D cg c]  }|d   j                  d      |k(  s| }}|r|d   S |r||j                  d      ddddS t        |      7 {# t        $ r1 | j	                  |       d{  7  }|r	|ddd
}Y |S t        |      w xY w7 # t
        $ r Y w xY w7 c c}w w)z!File information about this path.r7   r  r   Trm  Nr   r  )r   r   rQ  r=   r   r  r  r   detail)r   r   r   r  _lsr  rT  r  r
  ro   r:   r  r   r  rz  )rK   r;   r   r   r   existsparent_pathparent_cacher<   r  path_generationr=   oexactout0s                  r4   _infozGCSFileSystem._info  sQ    ##D)d?	2 JJu4&kDJII


4 Jll4(**;7'+t'<$_)*oF
88VSM*D! V9##C(D0"aeeL&9Z&GH	
 t$ !C( +# 	**400EV}E&M$:$:3$? %@  xDHHT,V,,@a!F)"2"23"74"?@@7N !C( +#  $D))o J 2#xx~--#'KHC J ,D112> 1 ! 		 -@s   HF1 F.F1 A6HH-HG0 G.G0 :HG?H H=H-H.F1 1G+GG+HG++H.G0 0	G<9H;G<<HHc                   K   | j                  |      j                  d      }|r| j                  |       |dv r| j                          d{   }n\g } | j                  |f||d| d{   D ]9  }|r$d|v r |j                         }|d    d|d    |d<   |j                  |       ; |r|S t        |D 	cg c]  }	|	d   	 c}	      S 7 7 bc c}	w w)z6List objects under the given '/{bucket}/{prefix} path.r7   )r7   r$   Nr  r   r=   #)r   r  rV  r   r(  copyr9  r8  )
rK   r;   r  r  r	  refreshr   r   entryr  s
             r4   r  zGCSFileSystem._ls  s      ##D)005!!$'9**,,CC1t11 #h :@   "  5!JJLE',V}oQu\7J6K$LE&M

5!" Jc21V9233 - 3s7   ACC C.C/AC<CCCCc                     d}| j                  |      \  }}}t        |      }|j                  | j                  |||r	d|       S d      S )zGet HTTP URL of the given pathz,{}/download/storage/v1/b/{}/o/{}?alt=media{}z&generation=r$   )r
  r   rk   r_   )rK   r;   ur<   rh  r   s         r4   r   zGCSFileSystem.url  s^    :%)__T%:"
vxxNN+5l:,'	
 	
 <>	
 	
r5   c                    K   | j                  |      }|s|rd| j                  |||       d{   i}ni }| j                  d||       d{   \  }}|S 7 )7 w)z Simple one-shot get of file dataRangeNr  )r   )r   _process_limitsr   )	rK   r;   startendr   u2headr   r   s	            r4   	_cat_filezGCSFileSystem._cat_file  sf     XXd^C4#7#7eS#IIJDD!ZZr4Z@@
	 J As!   -AAAA
AAc                 j   K   | j                  |       d{   j                  di       }||   S 7 w)z#Get user-defined metadata attributeNmetadata)r  r   )rK   r;   attrmetas       r4   	_getxattrzGCSFileSystem._getxattr)  s3     jj&&++J;Dz 's   313c           	        K   d|i}|||d<   |t         j                  d       ||d<   |j                  t        |             | j	                  |      \  }}}	| j                  dd||d|d	       d{   }
|
j                  di       S 7 w)
an  Set/delete/add writable metadata attributes

        Note: uses PATCH method (update), leaving unedited keys alone.
        fake-gcs-server:latest does not seem to support this.

        Parameters
        ----------
        content_type: str
            If not None, set the content-type to this value
        content_encoding: str
            This parameter is deprecated, you may use fixed_key_metadata instead.
            If not None, set the content-encoding.
            See https://cloud.google.com/storage/docs/transcoding
        fixed_key_metadata: dict
            Google metadata, in key/value pairs, supported keys:
                - cache_control
                - content_disposition
                - content_encoding
                - content_language
                - custom_time

            More info:
            https://cloud.google.com/storage/docs/metadata#mutable
        kw_args: key-value pairs like field="value" or field=None
            value must be string to add or modify, or None to delete

        Returns
        -------
        Entire metadata after update (even if only path is passed)
        r  NcontentTypez`setxattrs: content_encoding parameter is now deprecated you may use `fixed_key_metadata` insteadr&   r   r  T)fieldsr   r   )r   r   r   _convert_fixed_key_metadatar
  r   r   )rK   r;   content_typer+   fixed_key_metadatar   i_jsonr<   r  r   o_jsons              r4   
_setxattrszGCSFileSystem._setxattrs0  s     L f%#$0F=!'KK; )9F$%12DEF"&//$"7Zzz " 
 
 zz*b))
s   A2B4B5Bc                    K   | j                  |      \  }}}|D cg c]  }d| j                  |      d   i }}| j                  dd|||ddi|d||dd	
       d{    yc c}w 7 
w)z*Concatenate objects within a single bucketr=   r   r\  zb/{}/o/{}/composeContent-Typeapplication/jsonzstorage#composeRequest)r=   r<   )sourceObjectsrM  destination)destinationPredefinedAclr   r   N)r
  r   )	rK   r;   pathsrb  r<   r  r   r   sources	            r4   _mergezGCSFileSystem._mergeo  s     "&//$"7Z;@Aa64??1-a01AAjj%(#%78!'0(+v>  
 	
 	
 B	
s   A,A%&A,A* A,c                 |  K   | j                  |      \  }}}| j                  |      \  }}	}
|
rt        d      | j                  dd||||	ddi|d|
       d{   }|d	   dur1| j                  dd||||	ddi|d
   |d|       d{   }|d	   dur1| j                  | j	                  |             y7 ]7 /w)zDuplicate remote file*Cannot write to specific object generationr\  zb/{}/o/{}/rewriteTo/b/{}/o/{}r  r  T)r   r  r   sourceGenerationNdonerewriteToken)r   r  r  r   r  )r
  rj   r   rV  rT  )rK   path1path2rb  r   b1k1g1b2k2g2r   s               r4   _cp_filezGCSFileSystem._cp_file  s     __U+
B__U+
BIJJJJ+#%78%(  
 
 &k%

/');< 0),!# #  C &k% 	dll5125
s*   AB<B8/B<
B:B<"B<:B<c                   K   | j                  |      \  }}}|rR| j                  dd|||       d {    | j                  t        j                  | j                  |                   y | j                  |       d {    y 7 R7 w)Nrl  r  r   )r
  r   rV  r   dirnamer   rn  )rK   r;   r   r<   r  r   s         r4   _rm_filezGCSFileSystem._rm_file  s|     "&//$"7Z**X{FCJ*WWW!!)"3"3D4H4H4N"OP++d### X $s"   1B
BAB
 BB
B
c                    K   dd l }d}g }t        dd      D ]  }g }|}g }t        |      D ]\  \  }	}
 j                  |
      \  }}}|rd| nd}|j	                  |j                  |	dz   t        |      t        |      |             ^ dj                  |      } j                  d j                   d	d
di|dz          d {   \  }}|d
   j                  dd      d   }t         fd|D              t        |      z  }|D cg c]  } j                  |       c} |j                         }|j                  |      dd }t        ||      D ]  \  }}t        j                   d|      }|rt#        |j%                         d         nd }|dv r|j	                  |       S|t&        v r|dk  r|j	                  |       rt        j                   d|j)                  dd            }|r(t        j                   d|j%                         d         }nd }|r.|r,|j	                  t+        |j%                         d                |j	                  t+        t-        ||                    |rB|}t/        j0                  t3        |j                         d|dz
  z  z   d             d {     |S  |S 7 c c}w 7 w)Nr   a!  
--===============7330845974216740156==
Content-Type: application/http
Content-Transfer-Encoding: binary
Content-ID: <b29c5de2-0db4-490b-b421-6a51b598bd11+{i}>

DELETE /storage/v1/b/{bucket}/o/{key}{query} HTTP/1.1
Content-Type: application/json
accept: application/json
content-length: 0
r   rt   z?generation=r$   )re   r<   r  queryr\  z/batch/storage/v1r  z@multipart/mixed; boundary="===============7330845974216740156=="z)
--===============7330845974216740156==--r   r   =c              3   @   K   | ]  }j                  |        y wrA   )rT  )rV   r   rK   s     r4   rY   z*GCSFileSystem._rm_files.<locals>.<genexpr>  s     9a$,,q/9rZ   rw  zHTTP/[0-9.]+ ([0-9]+))         z{(.*)}
z({.*})r       )randomra   	enumerater
  r9  rk   r   r:   r   r_   splitrh   rV  rC   zipresearchr   groupsr   replacer  r   r   sleepmin)rK   r  r  templater   retry	remainingchunkpartsre   r   r<   r  r   r  bodyr   contentboundaryparentsparenttxt	responsesr;   responsemcodemsgmsg2s   `                            r4   	_rm_fileszGCSFileSystem._rm_files  s    < 	  1a[ 4	EIEE!%( 
1*.//!*<'Z7A,zl3rOOa%$V}!#J#	 $ 
 775>D%)ZZ>>""34" %2 HH &0 &  GW ~.44S!<Q?H9599CJFG9@AvT""6*A.."C		(+Ab1I"%eY"7 =hII5x@-.s188:a=)D:%JJt$T\eai$$T*))Hh.>.>tR.HIC!yy3::<?C#t

74;;=+;#<=

73tT?#;<!=" !mmC!	:J(JB$OPPP
k4	j 
K  B, Qs2   CKJ;AKJ>#FK.K/K>Kc                     d| j                   v S )Nztorage.googleapis.comr   rJ   s    r4   	on_googlezGCSFileSystem.on_google  s    &$..88r5   c                   K   | j                  |||       d {   }|D cg c]  }| j                  |      d   s| }}|D cg c]  }| j                  |      d   r| }}| j                  rat        t	        j
                  t        dt        |      |      D 	cg c]  }	| j                  ||	|	|z           c}	d       d {   g       }
n=t	        j
                  |D cg c]  }| j                  |       c}dd       d {   }
|
j                  t        j                  |D cg c]  }| j                  |       c}ddi d {          |
D cg c]1  }t        |t              rd	t!        |      vrt        |t"              s|3 }}|r|d   |
D cg c]!  }d	t!        |      vrt        |t"              s|# }
}|
st#        |      |
S 7 c c}w c c}w c c}	w 7 !c c}w 7 c c}w 7 c c}w c c}w w)
N)	recursivemaxdepthr   r   T)return_exceptionsr  )r  
batch_sizer  zNo such object)_expand_pathr
  r  sumr   _run_coros_in_chunksra   rb   r  r  r;  r   r:  rn  r   	Exceptionr   r  )rK   r;   r  r  	batchsizer  r   filesdirsre   exsfdexerrorss                  r4   _rmzGCSFileSystem._rm  s
    ''	H'UU!;qT__Q%7%:;; >a(:1(=>>>>// "'q#e*i!@ uQY'?@ '+  	C 11+01aq!1TVW C
 	

..*./Q$++a./CG 	
 
"i( B/r#45	 
 
 )O 
s2w.z"FW7X 
 

 #D))
W V;>
 2 0

s   HGHGGHG !G %:HG%=H	G*
H'G-?HG2"H/G4
HG9H6G;H!&H HH-H4H;
Hc           
        K   |xs | j                   }| j                  |      \  }}	}
t        |      }d }|dk  rt        | ||	|||||       d {   }nt	        | ||	|||       d {   }t        dt        |      |      D ]"  }||||z    }t        | |||||       d {   }$ t        |      }|j                  |       |j                  |       | j                  | j                  |             |S 7 7 7 Yw)Nr%   )r  r   )r   r
  rb   simple_uploadinitiate_uploadra   upload_chunkr   r   validate_json_responserV  rT  )rK   r;   r   r  r   r  r  	chunksizer<   r  r   r   r   rX  offsetbitcheckers                    r4   
_pipe_filezGCSFileSystem._pipe_file'  s+     "5T%5%5"&//$"7Z4y)*#5	 	H -#5 H  3t9i8 6FY$67((C|  .k:GNN4 **3/dll401=	s7   AC?C9C?*C;+8C?#C=$AC?;C?=C?c	                 l  K   t         j                  j                  |      ry |xs
 t               }|xs | j                  }t        |      }
| j                  |      \  }}}|rt        d      t        |d      5 }|j                  dd      }|j                  d       |j                  |       |dk  r;t        | |||j                         ||||       d {    |j                  |       nt        | |||||       d {   }d}	 |j                  |      }|snIt        | |||||       d {   }|t!        |      z  }|j                  |       |
j#                  |       ]|
j%                         | j'                  | j)                  |             d d d        y 7 7 7 s# 1 sw Y   y xY ww)Nr  rbr   r   r%   )r   
metadatainr  r  )r  r  )r\   r;   isdirr	   r   r   r
  rj   openrI   set_sizer  rB   absolute_updater  r  rb   r   r  rV  rT  )rK   lpathrpathr  r   r  r   callbackr  r   r  r<   r  r   f0r   rX  r  r  r   s                       r4   	_put_filezGCSFileSystem._put_fileW  s     77==-|~!5T%5%5)+6"&//%"8ZIJJ% )	7"771a=DGGAJd#i#GGI +'!-'9	 	 	 ((. "1 %'9"  ''),C ,hVT<! C c#h&F,,V4NN3'  ..s3!!$,,u"56S)	7 )	7	A)	7 )	7sV   A8F4:AF(F"*F(F$.F(2F&3A&F(	F4"F($F(&F((F1-F4c                 l   K   	 | j                  |       d {   d   dk(  S 7 
# t        $ r Y yw xY ww)Nr   r  F)r  r  rU  s     r4   _isdirzGCSFileSystem._isdir  s;     	**T**F3{BB* 		s(   4% #	% 4% 	1414c           	        K   | j                  |      }||dk  rt        d      | j                  |d||       d {   \  }}	|sX| j                  |      \  }
}}	|r|sdn|j	                  d      dz   }| | }n|}| j                  |
d||       d {   \  }}	i }i }|D ]  }| j                  |d         }|}|s| j                  |      d   }|rt        |      t        |      k  rJ|d|dd	dd
||<   |j                  |i       }|d   }||vr|||<   ||   }| j                  |      }|rs |sU|j                         D ci c]  \  }}|t        |j                               ! }}}| j                  j                  |       |r(t        |t        |j                               z   d       }|r;|j	                  d      j                  d      |z   t        t        fd|            }|r2|r|D ci c]  }|d    d|d    | c}S |D ci c]  }|d   |
 c}S |r|D cg c]  }|d    d|d     c}S |D cg c]  }|d   	 c}S 7 7 c c}}w c c}w c c}w c c}w c c}w w)Nr   zmaxdepth must be at least 1r$   )r+  r  r	  r7   r=   r   r  r  )KeySizer=   StorageClassr   r   c                     | d   S )Nr=   rQ   )xs    r4   <lambda>z%GCSFileSystem._find.<locals>.<lambda>  s
    !F) r5   )r  c                 2    | d   j                  d      k  S )Nr=   r7   )count)r  depths    r4   r  z%GCSFileSystem._find.<locals>.<lambda>  s    AfIOOC,@E,I r5   r  r   )r   rj   r!  r
  r  rT  rb   
setdefaultr   r   valuesr#  r   r8  r  filter)rK   r;   withdirsr  r  r	  r  r   objectsr>   r<   r  r5  _prefixr  cache_entriesobjr  previousdir_keyr  r=   r   r   cache_entries_listr  r  s                             @r4   _findzGCSFileSystem._find  s     ##D)HqL:;;  00Bv 1 
 

 !__T2NFC"%3::c?S+@"GF8,#44"Wx  5   JGQ  	.C\\#f+.FH//&1!4#f+D	"9 #"$/' V (2262>'w&$,GDM<f-) 		.4 BOBUBUBW!X$!Q!T!((*%5"5!X!XMM  !34WtDKKM'::@STGKK$**3/(:E6"I7STGELM1V9+Qq&78!;MM*12QAfIqL22>EFqyk1\?"34FF#*+a&	++A
D "Y N2 G+ss   <I:I AI:I(I:A2I:6I:$I 1BI:>I&I:I+'	I:0I0I:I5I:I: I:)r   r  c                `  K   |j                  d| j                        }| j                          d {    | j                  j	                  || j                  |      | j                  |      | j                        4 d {   }|j                          	 t        |j                  d         }	|j                  |	       t        |      }
t        j                   j#                  |      xs t        j$                  }t        j&                  |d       t)        |d      5 }	 |j*                  j-                  d       d {   }|sn=|j/                  |       |
j1                  |       |j3                  t5        |             c	 d d d        t7        |j8                  |       |
j;                  |       |j8                  |j                  |j<                  |fcd d d       d {    S 7 7 l# t        t        f$ r d }	Y Uw xY w7 # 1 sw Y   xY w7 1# 1 d {  7  sw Y   y xY ww)Nr   )r   r   r   r   zcontent-lengthT)exist_okwbi   )rl   r   r   r   r   r   r   r   raise_for_statusr   r   KeyErrorrj   r
  r   r\   r;   r  curdirmakedirsr	  r  rB   writer   relative_updaterb   r   r   validate_http_responser   )rK   r  r  r   r  rm   r   r   rD   r   r  lparentf2r   s                 r4   _get_file_requestzGCSFileSystem._get_file_request  s     jj0@0@A!!!<<####F+%%g.))	 $ 
 	= 	=
  199%567 d#-k:Gggooe,9		GKK$/eT" 8b!"	!::DHHTNNN4(,,SY7  	8 ahhe4**1-88QYY<7	= 	= 	= 	"	= j)  ;8 8!	= 	= 	= 	=s   0H.G+AH.G.H.HG10A0H  H H	AHAHH.%H&H..H.1HHHH	HH	HH.H+H" H+'H.c                    K   | j                  |      }t        j                  j                  |      ry |xs
 t	               } | j
                  ||fd|i| d {    y 7 w)Nr  )r   r\   r;   r  r	   r5  )rK   r  r  r  r   r  s         r4   	_get_filezGCSFileSystem._get_file  sT     XXe_77==-|~$d$$RLLVLLLs   AA#A!A#c           
      j    || j                   }|xs | j                  }t        | |||f||||||	d|S )zw
        See ``GCSFile``.

        consistency: None or str
            If None, use default for this instance
        )cache_optionsr   r  rb  
autocommitr  )r   r   GCSFile)rK   r;   moder   r9  rb  r   r  r:  r  r   r   consts                r4   _openzGCSFileSystem._open  sa    ( 00J/t//	

 (!1
 
 	
r5   c                    | j                  |      j                  d      }d|vr|ddfS |j                  dd      \  }}|}d}|rkt        |      }	 |j                  r|j                  }n-|j
                  r!t        |j
                        }d|v r|d   d   }|t        |       |j                  }|||fS # t        $ r d}Y w xY w)aH  
        Normalise GCS path string into bucket and key.

        Parameters
        ----------
        path : string
            Input path, like `gcs://mybucket/path/to/file`.
            Path is of the form: '[gs|gcs://]bucket[/key][?querystring][#fragment]'

        GCS allows object generation (object version) to be specified in either
        the URL fragment or the `generation` query parameter. When provided,
        the fragment will take priority over the `generation` query paramenter.

        Returns
        -------
            (bucket, key, generation) tuple
        r7   r$   Nr   r   r   )
r   lstripr  r   fragmentr  r   r   r;   rj   )	r   r;   r   r<   keypartr  r   r  parseds	            r4   r9   zGCSFileSystem._split_path>  s    & ""4(//4d?T>!**S!,
W%E">>!&J[[%ekk2F#v-%+L%9!%<
 )
O**C 
 	
  "!
"s   AB5 5CCc                 <    | j                  || j                        S )Nr   )r9   r   rU  s     r4   r
  zGCSFileSystem.split_pathn  s    D4F4FGGr5   c                 2   ddl m} |j                  | j                  j                  | j                        }| j                  |      \  }}}|j                  |      }|j                  |      }	 |	j                  dt        |      || j                  d|S )a6  Create a signed URL representing the given path.

        Parameters
        ----------
        path : str
             The path on the filesystem
        expiration : int
            Number of seconds to enable the URL for

        Returns
        -------
        URL : str
            The signed URL
        r   )storager   )seconds)
expirationr   api_access_endpointrQ   )google.cloudrF  Clientr   r   r
  r<   blobgenerate_signed_urlr   r   )
rK   r;   rH  r   rF  clientr<   r  r   rL  s
             r4   signzGCSFileSystem.signq  s     	)((44LL   

 #'//$"7Zv&{{3't'' 
 4! $
 	
 	
r5   )T)r$   F)Nr7   r$   FrA   )r    bucketOwnerFullControlNTFFN)Fr$   FFNN)NNN)FN   )NNapplication/octet-streamN   )NNrS  rT  NN)FFr$   FN)	r  NNNNNTNNF)d   )MrM   rN   rO   rP   r~   r   DEFAULT_BLOCK_SIZEr   r   
async_implDEFAULT_PROJECTr}   propertyr_   r   r   staticmethodr   r   r   classmethodr   r   r   r   r   r   r   r   r   sync_wrappercallr   r   r   make_bucket_requester_paysr  r(  r!  r2  r3  r   rV  rj  mkdirrn  rmdirrq  rt  r   r  r  r   r  r  getxattrr  	setxattrsr  merger  r  r  r  r  rmr  r  r  r(  r5  r7  r>  r9   r
  rO  __classcell__r   s   @r4   r8   r8      sg   hT 9FG+HJ  %4El - - / / ( (  "  
 ' '  	 7#+/d3 $30 -2E% 4U#D
 
2: "3!2!23N!O21h4n JO3jAF8t!>*. , %W&r Df%E"  Df%E((<<*~ FK42


 !t  +H
 ;*z "!!*-I
$ Df%E 3D$DL 9 9,\ 
		3	B /.h />7@ P,d 7#+/$ = $ =DM #
J -
 -
^H
r5   r8   c                   |     e Zd Zdeddddddddddf fd	Zed        Zd Zd	 Zdd
Z	d Z
d Zd Zd ZddZ xZS )r;  r  T	readaheadNmd5c                    |j                  |      \  }}}|st        d      t        ||      | _        t	        |   ||||f|||d| || _        || _        || _        || _	        t        |	      | _        d| j                  v r| j                  }ni }|xs |j                  dd      | _        |
xs |j                  di       | _        t#        |d      | _        | j$                  j'                  |xs i        || _        |d	k(  r;| j*                  t,        k  r t/        j0                  d
       t,        | _        d| _        yy)au  
        Open a file.

        Parameters
        ----------
        gcsfs: instance of GCSFileSystem
        path: str
            location in GCS, like 'bucket/path/to/file'
        mode: str
            Normal file modes. Currently only 'wb' amd 'rb'.
        block_size: int
            Buffer size for reading or writing
        acl: str
            ACL to apply, if any, one of ``ACLs``. New files are normally
            "bucketownerfullcontrol", but a default can be configured per
            bucket.
        consistency: str, 'none', 'size', 'md5', 'crc32c'
            Check for success in writing, applied at file close.
            'size' ensures that the number of bytes reported by GCS matches
            the number we wrote; 'md5' does a full checksum. Any value other
            than 'size' or 'md5' or 'crc32c' is assumed to mean no checking.
        content_type: str
            default is `application/octet-stream`. See the list of available
            content types at https://www.iana.org/assignments/media-types/media-types.txt
        metadata: dict
            Custom metadata, in key/value pairs, added at file creation
        fixed_key_metadata: dict
            Google metadata, in key/value pairs, supported keys:
                - cache_control
                - content_disposition
                - content_encoding
                - content_language
                - custom_time
            More info:
            https://cloud.google.com/storage/docs/metadata#mutable
        timeout: int
            Timeout seconds for the asynchronous callback.
        generation: str
            Object generation.
        zAttempt to open a bucket)r:  
cache_typer9  rD   r  rS  r  Tfrom_googler+  z*Setting block size to minimum value, 2**18N)r
  r  ro   r   r|   r}   r   r<   r  rb  r   r  r<  detailsr   r  r  r  r  r   r   	blocksizeGCS_MIN_BLOCK_SIZEr   r   rX  )rK   r   r;   r<  r   r:  rl  r9  rb  r   r  r  r   r  r   r   r<   r  r  detr   s                       r4   r}   zGCSFile.__init__  sL   t (-'7'7'=$_455.z?K			

 "!'		
 		
 
.{;$)),,CC( 
CGG5-
 !;CGGJ$;"=ct"T&&'9'?R@4<~~ 22JK!3 DM	 r5   c                     | j                   6| j                  j                  | j                  | j                        | _         | j                   S )Nr  )_detailsfsr   r;   r   rJ   s    r4   ro  zGCSFile.details  s6    ==  GGLLtLODM}}r5   c                     | j                   S )z File information about this path)ro  rJ   s    r4   r   zGCSFile.info  s    ||r5   c                 L    | j                   j                  | j                        S )zHTTP link to this file's data)ru  r   r;   rJ   s    r4   r   zGCSFile.url   s    ww{{499%%r5   c                    	 | j                   j                         }i }t        |      }|t        k  r|r| j                  syt        |t              }|d| }|r[| j                  rO||k(  rJ|r3d| j                  | j                  |z   dz
  | j                  |z   fz  |d<   n9d| j                  z  |d<   d}n$d| j                  | j                  |z   dz
  fz  |d<   |j                  | j                  t        |      d	       | j                  j                  d
| j                  ||      \  }}d|v rt        |d   j                  d      d         }	| j                  |z   dz
  |	z
  }
|
dkD  rh| j                   j                  |d|
         t#        ||
 d       | _         | j                   j%                  |
       | xj                  ||
z
  z  c_        | j                   j                  |       nV|sJ d       |rKt'        j(                  |      }| j                   j                  |       | j                   j+                  |       t#               | _         | xj                  |z  c_        	 y)zWrite one part of a multi-block file upload

        Parameters
        ----------
        final: bool
            Complete and commit upload
        TFNbytes %i-%i/%ir   Content-Rangez
bytes */%izbytes %i-%i/*r  zContent-Lengthr\  r  r  -r   z"Response looks like upload is over)buffergetvaluerb   rq  r:  r  GCS_MAX_BLOCK_SIZEr  r   r  r   r   r^  rX  r   r  r  rG   rI   r   r   r  )rK   finalr   r  lchunk_lengthr  r   r   r  	shortfalljs               r4   _upload_chunkzGCSFile._upload_chunk  sF    ;;'')DDD	A&&doo
  q"45L,'E\Q->,<l2Q6a@ -D) -94;;,FD)D(7KKKK,.2; )_% KK!%!2!2c,FWX !%

t% !0 !GX '!''*005a89![[1_q0C7	q=LL''[yj(9:"3D)4E"FDKKK$$Y/KK1y=0KLL''-BBBu

8,ALL''-LL77:+-DKKK1Kr5   c                 6    d| _         | j                  d       y)z%If not auto-committing, finalize fileT)r  N)r:  r  rJ   s    r4   commitzGCSFile.commitJ  s    &r5   c                    t        j                  | j                  j                  t        | j                  | j
                  | j                  | j                  | j                  | j                  | j                  	      | _        y)zCreate multi-uploadr   N)r   r   r   r{   r  r<   r  r  r  r  r   rX  rJ   s    r4   _initiate_uploadzGCSFile._initiate_uploadO  sU    		JJOOJJKKHHMM##LL

r5   c                     | j                   y| j                  j                  d| j                          d| _         d| _        y)zkCancel in-progress multi-upload

        Should only happen during discarding this write-mode file
        Nrl  T)rX  r   r^  closedrJ   s    r4   discardzGCSFile.discard]  s;    
 == 

MM	
 r5   c                 ~   | j                   j                  d       | j                   j                         }t        j                  | j
                  j                  t        | j
                  | j                  | j                  || j                  | j                  | j                  | j                  | j                         y)zOne-shot upload, less than 5MBr   r   N)r}  rI   rB   r   r   r   r{   r  r<   r  r  r   r  r  r   )rK   r   s     r4   _simple_uploadzGCSFile._simple_uploadk  s    {{!		JJOOJJKKHHMM##LL	
r5   c                     	 | j                   j                  | j                  ||      S # t        $ r}dt	        |      v rY d}~y d}~ww xY w)zvGet data from GCS

        start, end : None or integers
            if not both None, fetch only given range
        )r  r  znot satisfiableNr5   )r   cat_filer;   r   r   )rK   r  r  r  s       r4   _fetch_rangezGCSFile._fetch_range}  sI    	::&&tyy3&GG 	 CF*	s   '* 	AAAArU  rQ  )rM   rN   rO   rW  r}   rZ  ro  r   r   r  r  r  r  r  r  rf  rg  s   @r4   r;  r;    sn    
 %]!~  
&DL'


$r5   r;  Frm  c                |    i }| |S t         j                         D ]  \  }}|s|n|}|s|n|}|| v s| |   ||<     |S )a  
    Convert fixed_key_metadata to/from GCS format.

    Google uses camelCase for its parameters, this function transform
    exposed fixed_key_metadata (snake_case) to or from GCS(google) format

    Parameters
    ----------
    metadata: dict
        A key value pair of fixed_key_metadata, key can be either
        camel case or snake case.
    from_google: bool
        True means that the metadata come from google and thus should be converted
        to snake_case
    )SUPPORTED_FIXED_KEY_METADATAr   )r  rn  r   r  attribute_namesrcdsts          r4   r  r    s^      C
;AAC %^$c.$/nS(?}CH	%
 Jr5   c                   K   i }t        |      }d|||z   dz
  |fz  }||d<   |j                  |t        |      d       | j                  d||t	        |             d {   \  }	}
d|	v rIt        |	d   j                  d      d         }||z   dz
  |z
  }|rt        | ||| d  |||       d {   S |
rt        j                  |
      S d S 7 m7 w)	Nry  r   rz  r{  r\  r  r  r|  )
rb   r   r   r   rG   r   r  r  r   r   )ru  rX  r   r  r   r  r  r  ra   r   r  r  r  s                r4   r  r    s     DD	A
Q==E!DKKQHI$->t-D "  LGS '''"((-a01aZ!^s*	%HdI:;/dL   "4::c?+t+s%   AC!C"AC1C2CCc                 j  K   d|i}|r||d<   |j                  t        |             | j                  d| j                   dt	        |       dd|d|i       d {   \  }}|d	   }	t        |	t              r|	d
   n|	}
t        t        |	            dk  rt        j                  d|z         |
S 7 Tw)Nr=   r  r\  /upload/storage/v1/b//o	resumablezX-Upload-Content-Type)r   r;   
uploadTyper   r   Locationr   rR  zLocation failed: %s)r   r  r   r_   r   r   r   rb   r   r   error)ru  r<   r  r  r  r  r  r   r>   locr   s              r4   r  r    s      
A *HH();<=xx~25=/D(,7    JGQ *
CsD)#a&sC
3s8}r*W45Js   AB3B1AB3c           	        K   t        |      }| j                   dt        |       d}	d|i}
|||
d<   |
j                  t	        |             t        j                  |
      }
d|
z   dz   d| z   dz   }|j                         |z   d	z   }| j                  d
|	dddit        |      d       d {   }|j                  |       |j                  |       y 7 'w)Nr  r  r=   r  z7--==0==
Content-Type: application/json; charset=UTF-8

z
--==0==z
Content-Type: z

s
   
--==0==--r\  	multipartr  z#multipart/related; boundary="==0=="T)r  r   r   r   )r   r_   r   r   r  r   dumpsencoder   rG   r  )ru  r<   r  datainr  r   r  r  r  r;   r  r  r   r  s                 r4   r  r    s     &k2Gll^0vrBD}H)OO/0BCDzz(#H		'	(,<\N*K	LNT	U  ??v%6Dhh!FGt$   	A NN6""1%	s   B-C/C0(C)rS  NN)NNrS  N)CrP   r   ior   loggingr\   r   r  r   r   r   r   urllib.parser   r   r2   r   r   r   fsspec.callbacksr	   fsspec.implementations.httpr
   fsspec.utilsr   r   r$   r   r   checkersr   r   r   inventory_reportr   r  r   r   r   	getLoggerr   environr]   ACLsbACLsrY  rq  r  rW  r  r?   rE   BytesIOrG   r_   rf   ro   AsyncFileSystemr8   load_tokensspecAbstractBufferedFiler;  r  r  r  r  rQ   r5   r4   <module>r     sv    	   	  	   ( ! . !   ) 2 6 $ - * - 9 9			7	# BJJyryy'?@	 "))3R8    *#/)  $"$)


 ,(! v
D(( v
r/     rfkk.. rj :? 8,. ,> +!&r5   