%
% API Documentation for cssutils
% Module cssutils.tokenize2
%
% Generated by epydoc 3.0.1
% [Fri Feb 01 19:05:22 2008]
%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                          Module Description                           %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

    \index{cssutils \textit{(package)}!cssutils.tokenize2 \textit{(module)}|(}
\section{Module cssutils.tokenize2}

    \label{cssutils:tokenize2}

New CSS Tokenizer (a generator)
\textbf{Version:} \$LastChangedRevision: 300 \$



\textbf{Date:} \$LastChangedDate: 2007-09-01 15:55:42 +0200 (Sa, 01 Sep 2007) \$



\textbf{Author:} \$LastChangedBy: cthedot \$




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                           Class Description                           %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

    \index{cssutils \textit{(package)}!cssutils.cssproductions \textit{(module)}!cssutils.cssproductions.CSSProductions \textit{(class)}|(}
\subsection{Class CSSProductions}

    \label{cssutils:cssproductions:CSSProductions}
\begin{tabular}{cccccc}
% Line for object, linespec=[False]
\multicolumn{2}{r}{\settowidth{\BCL}{object}\multirow{2}{\BCL}{object}}
&&
  \\\cline{3-3}
  &&\multicolumn{1}{c|}{}
&&
  \\
&&\multicolumn{2}{l}{\textbf{cssutils.cssproductions.CSSProductions}}
\end{tabular}


most attributes are set later

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                Methods                                %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

  \subsubsection{Methods}


\large{\textbf{\textit{Inherited from object}}}

\begin{quote}
\_\_delattr\_\_(), \_\_getattribute\_\_(), \_\_hash\_\_(), \_\_init\_\_(), \_\_new\_\_(), \_\_reduce\_\_(), \_\_reduce\_ex\_\_(), \_\_repr\_\_(), \_\_setattr\_\_(), \_\_str\_\_()
\end{quote}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                              Properties                               %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

  \subsubsection{Properties}

    \vspace{-1cm}
\hspace{\varindent}\begin{longtable}{|p{\varnamewidth}|p{\vardescrwidth}|l}
\cline{1-2}
\cline{1-2} \centering \textbf{Name} & \centering \textbf{Description}& \\
\cline{1-2}
\endhead\cline{1-2}\multicolumn{3}{r}{\small\textit{continued on next page}}\\\endfoot\cline{1-2}
\endlastfoot\multicolumn{2}{|l|}{\textit{Inherited from object}}\\
\multicolumn{2}{|p{\varwidth}|}{\raggedright \_\_class\_\_}\\
\cline{1-2}
\end{longtable}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                            Class Variables                            %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

  \subsubsection{Class Variables}

    \vspace{-1cm}
\hspace{\varindent}\begin{longtable}{|p{\varnamewidth}|p{\vardescrwidth}|l}
\cline{1-2}
\cline{1-2} \centering \textbf{Name} & \centering \textbf{Description}& \\
\cline{1-2}
\endhead\cline{1-2}\multicolumn{3}{r}{\small\textit{continued on next page}}\\\endfoot\cline{1-2}
\endlastfoot\raggedright E\-O\-F\- & \raggedright \textbf{Value:} 
{\tt True}&\\
\cline{1-2}
\raggedright A\-T\-K\-E\-Y\-W\-O\-R\-D\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{ATKEYWORD}\texttt{'}}&\\
\cline{1-2}
\raggedright B\-O\-M\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{BOM}\texttt{'}}&\\
\cline{1-2}
\raggedright C\-D\-C\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{CDC}\texttt{'}}&\\
\cline{1-2}
\raggedright C\-D\-O\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{CDO}\texttt{'}}&\\
\cline{1-2}
\raggedright C\-H\-A\-R\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{CHAR}\texttt{'}}&\\
\cline{1-2}
\raggedright C\-H\-A\-R\-S\-E\-T\-\_\-S\-Y\-M\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{CHARSET\_SYM}\texttt{'}}&\\
\cline{1-2}
\raggedright C\-O\-M\-M\-E\-N\-T\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{COMMENT}\texttt{'}}&\\
\cline{1-2}
\raggedright D\-A\-S\-H\-M\-A\-T\-C\-H\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{DASHMATCH}\texttt{'}}&\\
\cline{1-2}
\raggedright D\-I\-M\-E\-N\-S\-I\-O\-N\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{DIMENSION}\texttt{'}}&\\
\cline{1-2}
\raggedright F\-O\-N\-T\-\_\-F\-A\-C\-E\-\_\-S\-Y\-M\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{FONT\_FACE\_SYM}\texttt{'}}&\\
\cline{1-2}
\raggedright F\-U\-N\-C\-T\-I\-O\-N\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{FUNCTION}\texttt{'}}&\\
\cline{1-2}
\raggedright H\-A\-S\-H\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{HASH}\texttt{'}}&\\
\cline{1-2}
\raggedright I\-D\-E\-N\-T\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{IDENT}\texttt{'}}&\\
\cline{1-2}
\raggedright I\-M\-P\-O\-R\-T\-\_\-S\-Y\-M\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{IMPORT\_SYM}\texttt{'}}&\\
\cline{1-2}
\raggedright I\-N\-C\-L\-U\-D\-E\-S\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{INCLUDES}\texttt{'}}&\\
\cline{1-2}
\raggedright I\-N\-V\-A\-L\-I\-D\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{INVALID}\texttt{'}}&\\
\cline{1-2}
\raggedright M\-E\-D\-I\-A\-\_\-S\-Y\-M\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{MEDIA\_SYM}\texttt{'}}&\\
\cline{1-2}
\raggedright N\-A\-M\-E\-S\-P\-A\-C\-E\-\_\-S\-Y\-M\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{NAMESPACE\_SYM}\texttt{'}}&\\
\cline{1-2}
\raggedright N\-U\-M\-B\-E\-R\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{NUMBER}\texttt{'}}&\\
\cline{1-2}
\raggedright P\-A\-G\-E\-\_\-S\-Y\-M\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{PAGE\_SYM}\texttt{'}}&\\
\cline{1-2}
\raggedright P\-E\-R\-C\-E\-N\-T\-A\-G\-E\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{PERCENTAGE}\texttt{'}}&\\
\cline{1-2}
\raggedright P\-R\-E\-F\-I\-X\-M\-A\-T\-C\-H\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{PREFIXMATCH}\texttt{'}}&\\
\cline{1-2}
\raggedright S\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{S}\texttt{'}}&\\
\cline{1-2}
\raggedright S\-T\-R\-I\-N\-G\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{STRING}\texttt{'}}&\\
\cline{1-2}
\raggedright S\-U\-B\-S\-T\-R\-I\-N\-G\-M\-A\-T\-C\-H\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{SUBSTRINGMATCH}\texttt{'}}&\\
\cline{1-2}
\raggedright S\-U\-F\-F\-I\-X\-M\-A\-T\-C\-H\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{SUFFIXMATCH}\texttt{'}}&\\
\cline{1-2}
\raggedright U\-N\-I\-C\-O\-D\-E\-\_\-R\-A\-N\-G\-E\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{UNICODE-RANGE}\texttt{'}}&\\
\cline{1-2}
\raggedright U\-R\-I\- & \raggedright \textbf{Value:} 
{\tt \texttt{'}\texttt{URI}\texttt{'}}&\\
\cline{1-2}
\end{longtable}

    \index{cssutils \textit{(package)}!cssutils.cssproductions \textit{(module)}!cssutils.cssproductions.CSSProductions \textit{(class)}|)}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                           Class Description                           %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

    \index{cssutils \textit{(package)}!cssutils.tokenize2 \textit{(module)}!cssutils.tokenize2.Tokenizer \textit{(class)}|(}
\subsection{Class Tokenizer}

    \label{cssutils:tokenize2:Tokenizer}
\begin{tabular}{cccccc}
% Line for object, linespec=[False]
\multicolumn{2}{r}{\settowidth{\BCL}{object}\multirow{2}{\BCL}{object}}
&&
  \\\cline{3-3}
  &&\multicolumn{1}{c|}{}
&&
  \\
&&\multicolumn{2}{l}{\textbf{cssutils.tokenize2.Tokenizer}}
\end{tabular}

\begin{description}
\item[{generates a list of Token tuples:}] \leavevmode 
(Tokenname, value, startline, startcolumn)

\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                Methods                                %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

  \subsubsection{Methods}

    \vspace{0.5ex}

\hspace{.8\funcindent}\begin{boxedminipage}{\funcwidth}

    \raggedright \textbf{\_\_init\_\_}(\textit{self}, \textit{macros}={\tt None}, \textit{productions}={\tt None})

    \vspace{-1.5ex}

    \rule{\textwidth}{0.5\fboxrule}
\setlength{\parskip}{2ex}

inits tokenizer with given macros and productions which default to
cssutils own macros and productions
\setlength{\parskip}{1ex}
      Overrides: object.\_\_init\_\_

    \end{boxedminipage}

    \label{cssutils:tokenize2:Tokenizer:tokenize}
    \index{cssutils \textit{(package)}!cssutils.tokenize2 \textit{(module)}!cssutils.tokenize2.Tokenizer \textit{(class)}!cssutils.tokenize2.Tokenizer.tokenize \textit{(method)}}

    \vspace{0.5ex}

\hspace{.8\funcindent}\begin{boxedminipage}{\funcwidth}

    \raggedright \textbf{tokenize}(\textit{self}, \textit{text}, \textit{fullsheet}={\tt False})

    \vspace{-1.5ex}

    \rule{\textwidth}{0.5\fboxrule}
\setlength{\parskip}{2ex}

generator: tokenizes text and yiels tokens, each token is a tuple of:
\begin{quote}{\ttfamily \raggedright \noindent
(tokenname,~tokenvalue,~line,~col)
}\end{quote}

The tokenvalue will contain a normal string, meaning CSS unicode
escapes have been resolved to normal characters. The serializer
escapes needed characters back to unicode escapes depending of
the stylesheet target encoding.
\begin{description}
\item[{text}] \leavevmode 
to be tokenized

\item[{fullsheet}] \leavevmode 
if \texttt{True} appends EOF token as last one and completes incomplete
COMMENT tokens

\end{description}
\setlength{\parskip}{1ex}
    \end{boxedminipage}


\large{\textbf{\textit{Inherited from object}}}

\begin{quote}
\_\_delattr\_\_(), \_\_getattribute\_\_(), \_\_hash\_\_(), \_\_new\_\_(), \_\_reduce\_\_(), \_\_reduce\_ex\_\_(), \_\_repr\_\_(), \_\_setattr\_\_(), \_\_str\_\_()
\end{quote}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                              Properties                               %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

  \subsubsection{Properties}

    \vspace{-1cm}
\hspace{\varindent}\begin{longtable}{|p{\varnamewidth}|p{\vardescrwidth}|l}
\cline{1-2}
\cline{1-2} \centering \textbf{Name} & \centering \textbf{Description}& \\
\cline{1-2}
\endhead\cline{1-2}\multicolumn{3}{r}{\small\textit{continued on next page}}\\\endfoot\cline{1-2}
\endlastfoot\multicolumn{2}{|l|}{\textit{Inherited from object}}\\
\multicolumn{2}{|p{\varwidth}|}{\raggedright \_\_class\_\_}\\
\cline{1-2}
\end{longtable}

    \index{cssutils \textit{(package)}!cssutils.tokenize2 \textit{(module)}!cssutils.tokenize2.Tokenizer \textit{(class)}|)}
    \index{cssutils \textit{(package)}!cssutils.tokenize2 \textit{(module)}|)}
