a
    •˜ëjüb  ã                   @  st   d dl mZ d dlZd dlmZmZmZmZ d dlm	Z	m
Z
 G dd„ de
ƒZdddd	d
œdd„ZG dd„ de
ƒZdS )é    )ÚannotationsN)ÚAnyÚLiteralÚOptionalÚUnion)ÚLanguageÚTextSplitterc                      s>   e Zd ZdZddddddœ‡ fd	d
„Zdddœdd„Z‡  ZS )ÚCharacterTextSplitterz(Splitting text that looks at characters.ú

FÚstrÚboolr   ÚNone)Ú	separatorÚis_separator_regexÚkwargsÚreturnc                   s"   t ƒ jf i |¤Ž || _|| _dS )úCreate a new TextSplitter.N)ÚsuperÚ__init__Ú
_separatorÚ_is_separator_regex)Úselfr   r   r   ©Ú	__class__© úa/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_text_splitters/character.pyr      s    zCharacterTextSplitter.__init__ú	list[str]©Útextr   c                   sh   ˆ j rˆ jn
t ˆ j¡}t||ˆ jd�}d}ˆ j oFt‡ fdd„|D ƒƒ}d}ˆ js\|s\ˆ j}ˆ  ||¡S )z=Split into chunks without re-inserting lookaround separators.©Úkeep_separator)z(?=z(?<!z(?<=z(?!c                 3  s   | ]}ˆ j  |¡V  qd S )N)r   Ú
startswith)Ú.0Úp©r   r   r   Ú	<genexpr>%   s   z3CharacterTextSplitter.split_text.<locals>.<genexpr>Ú )r   r   ÚreÚescapeÚ_split_text_with_regexÚ_keep_separatorÚanyÚ_merge_splits)r   r   Zsep_patternÚsplitsZlookaround_prefixesZis_lookaroundZ	merge_sepr   r$   r   Ú
split_text   s    ÿÿÿ
z CharacterTextSplitter.split_text)r
   F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r.   Ú__classcell__r   r   r   r   r	   	   s
     ýr	   r   ú$Union[bool, Literal['start', 'end']]r   )r   r   r    r   c                  sÐ   |rº|r¬t  d|› d�| ¡‰ |dkrF‡ fdd„tdtˆ ƒd dƒD ƒn‡ fd	d„tdtˆ ƒdƒD ƒ}tˆ ƒd dkr„|ˆ d
d … 7 }|dkrœg |¢ˆ d
 ‘nˆ d g|¢}qÂt  || ¡}nt| ƒ}dd„ |D ƒS )Nú(ú)Úendc                   s    g | ]}ˆ | ˆ |d    ‘qS ©é   r   ©r"   Úi©Z_splitsr   r   Ú
<listcomp>=   ó    z*_split_text_with_regex.<locals>.<listcomp>r   r9   é   c                   s    g | ]}ˆ | ˆ |d    ‘qS r8   r   r:   r<   r   r   r=   ?   r>   éÿÿÿÿc                 S  s   g | ]}|d kr|‘qS )r&   r   )r"   Úsr   r   r   r=   L   r>   )r'   ÚsplitÚrangeÚlenÚlist)r   r   r    r-   r   r<   r   r)   4   s     ÿ$ýÿýr)   c                      s|   e Zd ZdZdddddd	d
œ‡ fdd„Zddddœdd„Zdddœdd„Zeddd dœdd„ƒZe	dddœdd„ƒZ
‡  ZS )ÚRecursiveCharacterTextSplitterzŠSplitting text by recursively look at characters.

    Recursively tries to split by different characters to find one
    that works.
    NTFzOptional[list[str]]r4   r   r   r   )Ú
separatorsr    r   r   r   c                   s.   t ƒ jf d|i|¤Ž |p g d¢| _|| _dS )r   r    )r
   Ú
ú r&   N)r   r   Ú_separatorsr   )r   rG   r    r   r   r   r   r   r   V   s    z'RecursiveCharacterTextSplitter.__init__r   r   )r   rG   r   c                 C  s*  g }|d }g }t |ƒD ]P\}}| jr*|nt |¡}|dkrD|} qjt ||¡r|}||d d… } qjq| jrt|nt |¡}t||| jd�}	g }
| jrœdn|}|	D ]d}|  |¡| jk rÄ|
 	|¡ q¤|
râ|  
|
|¡}| |¡ g }
|sò| 	|¡ q¤|  ||¡}| |¡ q¤|
�r&|  
|
|¡}| |¡ |S )z&Split incoming text and return chunks.r@   r&   r9   Nr   )Ú	enumerater   r'   r(   Úsearchr)   r*   Z_length_functionZ_chunk_sizeÚappendr,   ÚextendÚ_split_text)r   r   rG   Zfinal_chunksr   Znew_separatorsr;   Z_sr   r-   Z_good_splitsrA   Zmerged_textZ
other_infor   r   r   rO   b   sB    ÿ

z*RecursiveCharacterTextSplitter._split_textr   c                 C  s   |   || j¡S )zíSplit the input text into smaller chunks based on predefined separators.

        Args:
            text (str): The input text to be split.

        Returns:
            List[str]: A list of text chunks obtained after splitting.
        )rO   rJ   )r   r   r   r   r   r.   Œ   s    	z)RecursiveCharacterTextSplitter.split_textr   )Úlanguager   r   c                 K  s   |   |¡}| f |ddœ|¤ŽS )aä  Return an instance of this class based on a specific language.

        This method initializes the text splitter with language-specific separators.

        Args:
            language (Language): The language to configure the text splitter for.
            **kwargs (Any): Additional keyword arguments to customize the splitter.

        Returns:
            RecursiveCharacterTextSplitter: An instance of the text splitter configured
            for the specified language.
        T)rG   r   )Úget_separators_for_language)ÚclsrP   r   rG   r   r   r   Úfrom_language—   s    
z,RecursiveCharacterTextSplitter.from_language)rP   r   c                 C  sl  | t jt jfv rg d¢S | t jkr*g d¢S | t jkr<g d¢S | t jkrNg d¢S | t jkr`g d¢S | t jkrrg d¢S | t jkr„g d¢S | t j	kr–g d¢S | t j
kr¨g d	¢S | t jkrºg d
¢S | t jkrÌg d¢S | t jkrÞg d¢S | t jkrðg d¢S | t jk�rg d¢S | t jk�rg d¢S | t jk�r,g d¢S | t jk�r@g d¢S | t jk�rTg d¢S | t jk�rhg d¢S | t jk�r|g d¢S | t jk�r�g d¢S | t jk�r¤g d¢S | t jk�r¸g d¢S | t jk�rÌg d¢S | t jk�r,d}d|› d�d|› d�d|› d�d|› d�d|› d�d d!d"d#d$d%d&d'd(d)d*d+d,gS | t jv �rLd-| › d.�}t|ƒ‚d-| › d/tt ƒ› �}t|ƒ‚d0S )1a
  Retrieve a list of separators specific to the given language.

        Args:
            language (Language): The language for which to get the separators.

        Returns:
            List[str]: A list of separators appropriate for the specified language.
        )ú
class z
void z
int z
float z
double ú
if ú
for ú
while ú
switch ú
case r
   rH   rI   r&   )ú
func ú
var ú
const ú
type rU   rV   rX   rY   r
   rH   rI   r&   )rT   ú
public ú
protected ú	
private ú
static rU   rV   rW   rX   rY   r
   rH   rI   r&   )rT   r^   r_   r`   z

internal z
companion z
fun ú
val r[   rU   rV   rW   z
when rY   ú
else r
   rH   rI   r&   )ú

function r\   ú
let r[   rT   rU   rV   rW   rX   rY   ú	
default r
   rH   rI   r&   )ú
enum ú
interface z
namespace r]   rT   rd   r\   re   r[   rU   rV   rW   rX   rY   rf   r
   rH   rI   r&   )rd   rT   rU   ú	
foreach rW   ú
do rX   rY   r
   rH   rI   r&   )
z	
message z	
service rg   z
option ú
import z
syntax r
   rH   rI   r&   )rT   ú
def z
	def r
   rH   rI   r&   )z
=+
z
-+
z
\*+
z

.. *

r
   rH   rI   r&   )rl   rT   rU   ú
unless rW   rV   rj   z
begin z
rescue r
   rH   rI   r&   )rl   z
defp z
defmodule z
defprotocol z

defmacro z
defmacrop rU   rm   rW   rY   z
cond z
with rV   rj   r
   rH   rI   r&   )z
fn r\   re   rU   rW   rV   z
loop ú
match r\   r
   rH   rI   r&   )rT   z
object rl   rb   r[   rU   rV   rW   rn   rY   r
   rH   rI   r&   )rZ   rT   ú
struct rg   rU   rV   rW   rj   rX   rY   r
   rH   rI   r&   )	z
#{1,6} z```
z	
\*\*\*+
z
---+
z
___+
r
   rH   rI   r&   )z
\\chapter{z
\\section{z
\\subsection{z
\\subsubsection{z
\\begin{enumerate}z
\\begin{itemize}z
\\begin{description}z
\\begin{list}z
\\begin{quote}z
\\begin{quotation}z
\\begin{verse}z
\\begin{verbatim}z
\\begin{align}z$$ú$rI   r&   )z<bodyz<divz<pz<brz<liz<h1z<h2z<h3z<h4z<h5z<h6z<spanz<tablez<trz<tdz<thz<ulz<olz<headerz<footerz<navz<headz<stylez<scriptz<metaz<titler&   )rh   rg   z
implements z

delegate ú
event rT   z

abstract r^   r_   r`   ra   z
return rU   z

continue rV   ri   rW   rX   z
break rY   rc   ú
try z
throw ú	
finally ú
catch r
   rH   rI   r&   )z
pragma z
using z

contract rh   z	
library z
constructor r]   rd   rq   z

modifier z
error ro   rg   rU   rV   rW   z

do while z

assembly r
   rH   rI   r&   )z
IDENTIFICATION DIVISION.z
ENVIRONMENT DIVISION.z
DATA DIVISION.z
PROCEDURE DIVISION.z
WORKING-STORAGE SECTION.z
LINKAGE SECTION.z
FILE SECTION.z
INPUT-OUTPUT SECTION.z
OPEN z
CLOSE z
READ z
WRITE z
IF z
ELSE z
MOVE z	
PERFORM z
UNTIL z	
VARYING z
ACCEPT z	
DISPLAY z

STOP RUN.rH   rI   r&   )
z
local rd   rU   rV   rW   z
repeat r
   rH   rI   r&   )z	
main :: z
main = re   z
in rj   z
where ú
:: z
= ú
data z	
newtype r]   ru   z
module rk   z
qualified z
import qualified rT   z

instance rY   z
| rv   z
= {z
, r
   rH   rI   r&   )rd   z
param rU   ri   rV   rW   rX   rT   rr   rt   rs   r
   rH   rI   r&   z*(?:Public|Private|Friend|Global|Static)\s+z\n(?!End\s)z?Sub\s+z?Function\s+z?Property\s+(?:Get|Let|Set)\s+z?Type\s+z?Enum\s+z\n(?!End\s)If\s+z\nElseIf\s+z	\nElse\s+z\nSelect\s+Case\s+z	\nCase\s+z\nFor\s+z\nDo\s+z
\nWhile\s+z	\nWith\s+z\n\nz\nrI   r&   z	Language z is not implemented yet!z& is not supported! Please choose from N)r   ÚCÚCPPZGOZJAVAZKOTLINZJSZTSZPHPÚPROTOÚPYTHONZRSTZRUBYZELIXIRZRUSTZSCALAZSWIFTZMARKDOWNZLATEXZHTMLZCSHARPZSOLZCOBOLZLUAZHASKELLZ
POWERSHELLZVISUALBASIC6Ú_value2member_map_Ú
ValueErrorrE   )rP   ZvisÚmsgr   r   r   rQ   ª   s–    












$&




ëÿz:RecursiveCharacterTextSplitter.get_separators_for_language)NTF)r/   r0   r1   r2   r   rO   r.   ÚclassmethodrS   ÚstaticmethodrQ   r3   r   r   r   r   rF   O   s      ü*rF   )Ú
__future__r   r'   Útypingr   r   r   r   Zlangchain_text_splitters.baser   r   r	   r)   rF   r   r   r   r   Ú<module>   s   +